From pypy.commits at gmail.com  Mon May  1 00:32:38 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 30 Apr 2017 21:32:38 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-obj-stealing: fix tests for
 PyList_SetItem reference stealing
Message-ID: <5906ba66.bbad500a.d900e.7d8f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-obj-stealing
Changeset: r91157:104e7d49fd08
Date: 2017-05-01 07:26 +0300
http://bitbucket.org/pypy/pypy/changeset/104e7d49fd08/

Log:	fix tests for PyList_SetItem reference stealing

diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -1872,6 +1872,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -1887,6 +1888,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -157,6 +157,8 @@
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
              PyList_SET_ITEM(o, 0, args);
              PyList_SET_ITEM(l, 1, args);
 

From pypy.commits at gmail.com  Mon May  1 00:36:34 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 30 Apr 2017 21:36:34 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-obj-stealing: merge default into branch
Message-ID: <5906bb52.dcb7500a.e7bfa.7d64@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-obj-stealing
Changeset: r91158:168652ad5d5c
Date: 2017-05-01 07:35 +0300
http://bitbucket.org/pypy/pypy/changeset/168652ad5d5c/

Log:	merge default into branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -30,3 +30,7 @@
 large constants is reused across instructions.
 
 .. branch: vmprof-0.4.4
+
+.. branch: controller-refactor
+
+Refactor rpython.rtyper.controllerentry.
diff --git a/rpython/rtyper/controllerentry.py b/rpython/rtyper/controllerentry.py
--- a/rpython/rtyper/controllerentry.py
+++ b/rpython/rtyper/controllerentry.py
@@ -1,6 +1,10 @@
+from rpython.flowspace.model import Constant
+from rpython.flowspace.operation import op
 from rpython.annotator import model as annmodel
 from rpython.tool.pairtype import pairtype
 from rpython.annotator.bookkeeper import getbookkeeper
+from rpython.rlib.objectmodel import specialize
+from rpython.rtyper.rmodel import Repr
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.rtyper.annlowlevel import cachedtype
 from rpython.rtyper.error import TyperError
@@ -10,7 +14,14 @@
 
     def compute_result_annotation(self, *args_s, **kwds_s):
         controller = self.getcontroller(*args_s, **kwds_s)
-        return controller.ctrl_new_ex(self.bookkeeper, *args_s, **kwds_s)
+        if kwds_s:
+            raise TypeError("cannot handle keyword arguments in %s" % (
+                self.new,))
+        s_real_obj = delegate(controller.new, *args_s)
+        if s_real_obj == annmodel.s_ImpossibleValue:
+            return annmodel.s_ImpossibleValue
+        else:
+            return SomeControlledInstance(s_real_obj, controller)
 
     def getcontroller(self, *args_s, **kwds_s):
         return self._controller_()
@@ -19,9 +30,10 @@
         if hop.s_result == annmodel.s_ImpossibleValue:
             raise TyperError("object creation always raises: %s" % (
                 hop.spaceop,))
+        assert not kwds_i
         controller = hop.s_result.controller
-        return controller.rtype_new(hop, **kwds_i)
-
+        return rtypedelegate(controller.new, hop, revealargs=[],
+            revealresult=True)
 
 
 def controlled_instance_box(controller, obj):
@@ -53,91 +65,25 @@
     def _freeze_(self):
         return True
 
+    @specialize.arg(0)
     def box(self, obj):
         return controlled_instance_box(self, obj)
-    box._annspecialcase_ = 'specialize:arg(0)'
 
+    @specialize.arg(0)
     def unbox(self, obj):
         return controlled_instance_unbox(self, obj)
-    unbox._annspecialcase_ = 'specialize:arg(0)'
 
+    @specialize.arg(0)
     def is_box(self, obj):
         return controlled_instance_is_box(self, obj)
-    is_box._annspecialcase_ = 'specialize:arg(0)'
 
-    def ctrl_new(self, *args_s, **kwds_s):
-        if kwds_s:
-            raise TypeError("cannot handle keyword arguments in %s" % (
-                self.new,))
-        s_real_obj = delegate(self.new, *args_s)
-        if s_real_obj == annmodel.s_ImpossibleValue:
-            return annmodel.s_ImpossibleValue
-        else:
-            return SomeControlledInstance(s_real_obj, controller=self)
-
-    def ctrl_new_ex(self, bookkeeper, *args_s, **kwds_s):
-        return self.ctrl_new(*args_s, **kwds_s)
-
-    def rtype_new(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.new, hop, revealargs=[], revealresult=True)
-
+    @specialize.arg(0, 2)
     def getattr(self, obj, attr):
         return getattr(self, 'get_' + attr)(obj)
-    getattr._annspecialcase_ = 'specialize:arg(0, 2)'
 
-    def ctrl_getattr(self, s_obj, s_attr):
-        return delegate(self.getattr, s_obj, s_attr)
-
-    def rtype_getattr(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.getattr, hop)
-
+    @specialize.arg(0, 2)
     def setattr(self, obj, attr, value):
         return getattr(self, 'set_' + attr)(obj, value)
-    setattr._annspecialcase_ = 'specialize:arg(0, 2)'
-
-    def ctrl_setattr(self, s_obj, s_attr, s_value):
-        return delegate(self.setattr, s_obj, s_attr, s_value)
-
-    def rtype_setattr(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.setattr, hop)
-
-    def ctrl_getitem(self, s_obj, s_key):
-        return delegate(self.getitem, s_obj, s_key)
-
-    def rtype_getitem(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.getitem, hop)
-
-    def ctrl_setitem(self, s_obj, s_key, s_value):
-        return delegate(self.setitem, s_obj, s_key, s_value)
-
-    def rtype_setitem(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.setitem, hop)
-
-    def ctrl_delitem(self, s_obj, s_key):
-        return delegate(self.delitem, s_obj, s_key)
-
-    def rtype_delitem(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.delitem, hop)
-
-    def ctrl_bool(self, s_obj):
-        return delegate(self.bool, s_obj)
-
-    def rtype_bool(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.bool, hop)
-
-    def ctrl_call(self, s_obj, *args_s):
-        return delegate(self.call, s_obj, *args_s)
-
-    def rtype_call(self, hop):
-        from rpython.rtyper.rcontrollerentry import rtypedelegate
-        return rtypedelegate(self.call, hop)
 
 
 def delegate(boundmethod, *args_s):
@@ -158,7 +104,6 @@
             return SomeControlledInstance(s_real_obj, controller=controller)
 
     def specialize_call(self, hop):
-        from rpython.rtyper.rcontrollerentry import ControlledInstanceRepr
         if not isinstance(hop.r_result, ControlledInstanceRepr):
             raise TyperError("box() should return ControlledInstanceRepr,\n"
                              "got %r" % (hop.r_result,))
@@ -176,7 +121,6 @@
             return s_obj.s_real_obj
 
     def specialize_call(self, hop):
-        from rpython.rtyper.rcontrollerentry import ControlledInstanceRepr
         if not isinstance(hop.args_r[1], ControlledInstanceRepr):
             raise TyperError("unbox() should take a ControlledInstanceRepr,\n"
                              "got %r" % (hop.args_r[1],))
@@ -219,41 +163,40 @@
         return SomeControlledInstance(self.s_real_obj, self.controller)
 
     def rtyper_makerepr(self, rtyper):
-        from rpython.rtyper.rcontrollerentry import ControlledInstanceRepr
         return ControlledInstanceRepr(rtyper, self.s_real_obj, self.controller)
 
     def rtyper_makekey(self):
         real_key = self.s_real_obj.rtyper_makekey()
         return self.__class__, real_key, self.controller
 
+    def getattr(self, s_attr):
+        assert s_attr.is_constant()
+        ctrl = self.controller
+        return delegate(ctrl.getattr, self.s_real_obj, s_attr)
 
-class __extend__(SomeControlledInstance):
+    def setattr(self, s_attr, s_value):
+        assert s_attr.is_constant()
+        ctrl = self.controller
+        return delegate(ctrl.setattr, self.s_real_obj, s_attr, s_value)
 
-    def getattr(s_cin, s_attr):
-        assert s_attr.is_constant()
-        return s_cin.controller.ctrl_getattr(s_cin.s_real_obj, s_attr)
+    def bool(self):
+        ctrl = self.controller
+        return delegate(ctrl.bool, self.s_real_obj)
 
-    def setattr(s_cin, s_attr, s_value):
-        assert s_attr.is_constant()
-        s_cin.controller.ctrl_setattr(s_cin.s_real_obj, s_attr, s_value)
-
-    def bool(s_cin):
-        return s_cin.controller.ctrl_is_true(s_cin.s_real_obj)
-
-    def simple_call(s_cin, *args_s):
-        return s_cin.controller.ctrl_call(s_cin.s_real_obj, *args_s)
+    def simple_call(self, *args_s):
+        return delegate(self.controller.call, self.s_real_obj, *args_s)
 
 
 class __extend__(pairtype(SomeControlledInstance, annmodel.SomeObject)):
 
     def getitem((s_cin, s_key)):
-        return s_cin.controller.ctrl_getitem(s_cin.s_real_obj, s_key)
+        return delegate(s_cin.controller.getitem, s_cin.s_real_obj, s_key)
 
     def setitem((s_cin, s_key), s_value):
-        s_cin.controller.ctrl_setitem(s_cin.s_real_obj, s_key, s_value)
+        delegate(s_cin.controller.setitem, s_cin.s_real_obj, s_key, s_value)
 
     def delitem((s_cin, s_key)):
-        s_cin.controller.ctrl_delitem(s_cin.s_real_obj, s_key)
+        delegate(s_cin.controller.delitem, s_cin.s_real_obj, s_key)
 
 
 class __extend__(pairtype(SomeControlledInstance, SomeControlledInstance)):
@@ -264,3 +207,75 @@
         return SomeControlledInstance(annmodel.unionof(s_cin1.s_real_obj,
                                                        s_cin2.s_real_obj),
                                       s_cin1.controller)
+
+class ControlledInstanceRepr(Repr):
+
+    def __init__(self, rtyper, s_real_obj, controller):
+        self.rtyper = rtyper
+        self.s_real_obj = s_real_obj
+        self.r_real_obj = rtyper.getrepr(s_real_obj)
+        self.controller = controller
+        self.lowleveltype = self.r_real_obj.lowleveltype
+
+    def convert_const(self, value):
+        real_value = self.controller.convert(value)
+        return self.r_real_obj.convert_const(real_value)
+
+    def reveal(self, r):
+        if r is not self:
+            raise TyperError("expected %r, got %r" % (self, r))
+        return self.s_real_obj, self.r_real_obj
+
+    def rtype_getattr(self, hop):
+        return rtypedelegate(self.controller.getattr, hop)
+
+    def rtype_setattr(self, hop):
+        return rtypedelegate(self.controller.setattr, hop)
+
+    def rtype_bool(self, hop):
+        return rtypedelegate(self.controller.bool, hop)
+
+    def rtype_simple_call(self, hop):
+        return rtypedelegate(self.controller.call, hop)
+
+
+class __extend__(pairtype(ControlledInstanceRepr, Repr)):
+
+    def rtype_getitem((r_controlled, r_key), hop):
+        return rtypedelegate(r_controlled.controller.getitem, hop)
+
+    def rtype_setitem((r_controlled, r_key), hop):
+        return rtypedelegate(r_controlled.controller.setitem, hop)
+
+    def rtype_delitem((r_controlled, r_key), hop):
+        return rtypedelegate(r_controlled.controller.delitem, hop)
+
+
+def rtypedelegate(callable, hop, revealargs=[0], revealresult=False):
+    bk = hop.rtyper.annotator.bookkeeper
+    c_meth = Constant(callable)
+    s_meth = bk.immutablevalue(callable)
+    hop2 = hop.copy()
+    for index in revealargs:
+        r_controlled = hop2.args_r[index]
+        if not isinstance(r_controlled, ControlledInstanceRepr):
+            raise TyperError("args_r[%d] = %r, expected ControlledInstanceRepr"
+                             % (index, r_controlled))
+        s_new, r_new = r_controlled.s_real_obj, r_controlled.r_real_obj
+        hop2.args_s[index], hop2.args_r[index] = s_new, r_new
+        v = hop2.args_v[index]
+        if isinstance(v, Constant):
+            real_value = r_controlled.controller.convert(v.value)
+            hop2.args_v[index] = Constant(real_value)
+    if revealresult:
+        r_controlled = hop2.r_result
+        if not isinstance(r_controlled, ControlledInstanceRepr):
+            raise TyperError("r_result = %r, expected ControlledInstanceRepr"
+                             % (r_controlled,))
+        s_new, r_new = r_controlled.s_real_obj, r_controlled.r_real_obj
+        hop2.s_result, hop2.r_result = s_new, r_new
+    hop2.v_s_insertfirstarg(c_meth, s_meth)
+    spaceop = op.simple_call(*hop2.args_v)
+    spaceop.result = hop2.spaceop.result
+    hop2.spaceop = spaceop
+    return hop2.dispatch()
diff --git a/rpython/rtyper/rcontrollerentry.py b/rpython/rtyper/rcontrollerentry.py
deleted file mode 100644
--- a/rpython/rtyper/rcontrollerentry.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from rpython.flowspace.model import Constant
-from rpython.flowspace.operation import op
-from rpython.rtyper.error import TyperError
-from rpython.rtyper.rmodel import Repr
-from rpython.tool.pairtype import pairtype
-
-
-class ControlledInstanceRepr(Repr):
-
-    def __init__(self, rtyper, s_real_obj, controller):
-        self.rtyper = rtyper
-        self.s_real_obj = s_real_obj
-        self.r_real_obj = rtyper.getrepr(s_real_obj)
-        self.controller = controller
-        self.lowleveltype = self.r_real_obj.lowleveltype
-
-    def convert_const(self, value):
-        real_value = self.controller.convert(value)
-        return self.r_real_obj.convert_const(real_value)
-
-    def reveal(self, r):
-        if r is not self:
-            raise TyperError("expected %r, got %r" % (self, r))
-        return self.s_real_obj, self.r_real_obj
-
-    def rtype_getattr(self, hop):
-        return self.controller.rtype_getattr(hop)
-
-    def rtype_setattr(self, hop):
-        return self.controller.rtype_setattr(hop)
-
-    def rtype_bool(self, hop):
-        return self.controller.rtype_bool(hop)
-
-    def rtype_simple_call(self, hop):
-        return self.controller.rtype_call(hop)
-
-
-class __extend__(pairtype(ControlledInstanceRepr, Repr)):
-
-    def rtype_getitem((r_controlled, r_key), hop):
-        return r_controlled.controller.rtype_getitem(hop)
-
-    def rtype_setitem((r_controlled, r_key), hop):
-        return r_controlled.controller.rtype_setitem(hop)
-
-    def rtype_delitem((r_controlled, r_key), hop):
-        return r_controlled.controller.rtype_delitem(hop)
-
-
-def rtypedelegate(callable, hop, revealargs=[0], revealresult=False):
-    bk = hop.rtyper.annotator.bookkeeper
-    c_meth = Constant(callable)
-    s_meth = bk.immutablevalue(callable)
-    hop2 = hop.copy()
-    for index in revealargs:
-        r_controlled = hop2.args_r[index]
-        if not isinstance(r_controlled, ControlledInstanceRepr):
-            raise TyperError("args_r[%d] = %r, expected ControlledInstanceRepr"
-                             % (index, r_controlled))
-        s_new, r_new = r_controlled.s_real_obj, r_controlled.r_real_obj
-        hop2.args_s[index], hop2.args_r[index] = s_new, r_new
-        v = hop2.args_v[index]
-        if isinstance(v, Constant):
-            real_value = r_controlled.controller.convert(v.value)
-            hop2.args_v[index] = Constant(real_value)
-    if revealresult:
-        r_controlled = hop2.r_result
-        if not isinstance(r_controlled, ControlledInstanceRepr):
-            raise TyperError("r_result = %r, expected ControlledInstanceRepr"
-                             % (r_controlled,))
-        s_new, r_new = r_controlled.s_real_obj, r_controlled.r_real_obj
-        hop2.s_result, hop2.r_result = s_new, r_new
-    hop2.v_s_insertfirstarg(c_meth, s_meth)
-    spaceop = op.simple_call(*hop2.args_v)
-    spaceop.result = hop2.spaceop.result
-    hop2.spaceop = spaceop
-    return hop2.dispatch()

From pypy.commits at gmail.com  Mon May  1 09:49:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 01 May 2017 06:49:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #1213: in ctypes,
 complain instead of silently ignoring _swappedbytes_
Message-ID: <59073d07.3ab8500a.c1b89.cded@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91159:fa2b7ecca667
Date: 2017-05-01 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/fa2b7ecca667/

Log:	Issue #1213: in ctypes, complain instead of silently ignoring
	_swappedbytes_

diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -234,6 +234,9 @@
         if ('_abstract_' in cls.__dict__ or cls is Structure 
                                          or cls is union.Union):
             raise TypeError("abstract class")
+        if hasattr(cls, '_swappedbytes_'):
+            raise NotImplementedError("missing in PyPy: structure/union with "
+                                      "swapped (non-native) byte ordering")
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self

From pypy.commits at gmail.com  Mon May  1 18:32:37 2017
From: pypy.commits at gmail.com (amauryfa@gmail.com)
Date: Mon, 01 May 2017 15:32:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Expose win32's SetErrorMode,
 which is mandatory for the 3.5 test suite.
Message-ID: <5907b785.0537ed0a.c9afd.bbf3@mx.google.com>

Author: amauryfa at gmail.com <Amaury Forgeot d'Arc>
Branch: 
Changeset: r91160:e69e22840104
Date: 2017-05-01 15:26 -0700
http://bitbucket.org/pypy/pypy/changeset/e69e22840104/

Log:	Expose win32's SetErrorMode, which is mandatory for the 3.5 test
	suite.

diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -83,6 +83,12 @@
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS     0x0001
+#define SEM_NOGPFAULTERRORBOX      0x0002 
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004 
+#define SEM_NOOPENFILEERRORBOX     0x8000 
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x53\x03\x00\x00\x13\x11\x00\x00\x56\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x52\x03\x00\x00\x51\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x45\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x55\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4F\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4C\x23GetStdHandle',0,b'\x00\x00\x42\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x3B\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3E\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x47\x23_getwch',0,b'\x00\x00\x47\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x49\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x44\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x51\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x52\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x45\x11wShowWindow',b'\x00\x00\x45\x11cbReserved2',b'\x00\x00\x54\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x51PROCESS_INFORMATION',b'\x00\x00\x00\x52STARTUPINFO',b'\x00\x00\x00\x45wint_t'),
 )

From pypy.commits at gmail.com  Mon May  1 18:40:39 2017
From: pypy.commits at gmail.com (amauryfa@gmail.com)
Date: Mon, 01 May 2017 15:40:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport _pypy_winbase_build.py from
 the 3.5 branch, to reduce merge conflicts.
Message-ID: <5907b967.6730ed0a.c75a0.ed71@mx.google.com>

Author: amauryfa at gmail.com <Amaury Forgeot d'Arc>
Branch: 
Changeset: r91161:aa98455c75dc
Date: 2017-05-01 15:39 -0700
http://bitbucket.org/pypy/pypy/changeset/aa98455c75dc/

Log:	Backport _pypy_winbase_build.py from the 3.5 branch, to reduce merge
	conflicts.

diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -79,16 +79,20 @@
 BOOL WINAPI CreateProcessA(char *, char *, void *,
                            void *, BOOL, DWORD, char *,
                            char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
+                           void *, BOOL, DWORD, wchar_t *,
+                           wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
 
 UINT WINAPI SetErrorMode(UINT);
 #define SEM_FAILCRITICALERRORS     0x0001
-#define SEM_NOGPFAULTERRORBOX      0x0002 
-#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004 
-#define SEM_NOOPENFILEERRORBOX     0x8000 
+#define SEM_NOGPFAULTERRORBOX      0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX     0x8000
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x53\x03\x00\x00\x13\x11\x00\x00\x56\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x52\x03\x00\x00\x51\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x45\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x00\x0F\x00\x00\x45\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x55\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4F\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4C\x23GetStdHandle',0,b'\x00\x00\x42\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x3B\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3E\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x47\x23_getwch',0,b'\x00\x00\x47\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x49\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x44\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x51\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x52\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x45\x11wShowWindow',b'\x00\x00\x45\x11cbReserved2',b'\x00\x00\x54\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x51PROCESS_INFORMATION',b'\x00\x00\x00\x52STARTUPINFO',b'\x00\x00\x00\x45wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
 )

From pypy.commits at gmail.com  Mon May  1 18:48:47 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Mon, 01 May 2017 15:48:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5907bb4f.4386370a.7d5a5.b4bc@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91162:7e5247599a57
Date: 2017-05-02 00:48 +0200
http://bitbucket.org/pypy/pypy/changeset/7e5247599a57/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -233,6 +233,9 @@
         if ('_abstract_' in cls.__dict__ or cls is Structure 
                                          or cls is union.Union):
             raise TypeError("abstract class")
+        if hasattr(cls, '_swappedbytes_'):
+            raise NotImplementedError("missing in PyPy: structure/union with "
+                                      "swapped (non-native) byte ordering")
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -76,6 +76,9 @@
 HANDLE WINAPI GetCurrentProcess(void);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
                            void *, BOOL, DWORD, wchar_t *,
                            wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
@@ -84,6 +87,12 @@
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
 DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS     0x0001
+#define SEM_NOGPFAULTERRORBOX      0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX     0x8000
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x13\x11\x00\x00\x59\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x16\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x13\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x4C\x03\x00\x00\x2D\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2D\x11\x00\x00\x2D\x11\x00\x00\x54\x03\x00\x00\x53\x03\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x2D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\x47\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x56\x03\x00\x00\x02\x01\x00\x00\x58\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x18\x23CloseHandle',0,b'\x00\x00\x12\x23CreatePipe',0,b'\x00\x00\x2C\x23CreateProcessW',0,b'\x00\x00\x23\x23DuplicateHandle',0,b'\x00\x00\x51\x23GetCurrentProcess',0,b'\x00\x00\x1F\x23GetExitCodeProcess',0,b'\x00\x00\x3F\x23GetModuleFileNameW',0,b'\x00\x00\x4E\x23GetStdHandle',0,b'\x00\x00\x44\x23GetVersion',0,b'\x00\x00\x1B\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x49\x23_getwch',0,b'\x00\x00\x49\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x4B\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x46\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x00\x54\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x55\x11lpReserved',b'\x00\x00\x55\x11lpDesktop',b'\x00\x00\x55\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\x47\x11wShowWindow',b'\x00\x00\x47\x11cbReserved2',b'\x00\x00\x57\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x36LPPROCESS_INFORMATION',b'\x00\x00\x00\x35LPSTARTUPINFO',b'\x00\x00\x00\x53PROCESS_INFORMATION',b'\x00\x00\x00\x54STARTUPINFO',b'\x00\x00\x00\x47wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
 )

From pypy.commits at gmail.com  Tue May  2 10:34:24 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:24 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Merge number of
 segments in use function
Message-ID: <590898f0.ce28c80a.40691.6129@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2046:439c2a69589d
Date: 2017-04-29 12:02 +0200
http://bitbucket.org/pypy/stmgc/changeset/439c2a69589d/

Log:	Merge number of segments in use function

diff --git a/c8/stm/sync.c b/c8/stm/sync.c
--- a/c8/stm/sync.c
+++ b/c8/stm/sync.c
@@ -176,6 +176,16 @@
 
 /************************************************************/
 
+static uint8_t number_of_segments_in_use(void) {
+    uint8_t result = 0;
+    int num;
+    for (num = 1; num < NB_SEGMENTS; num++) {
+        if (sync_ctl.in_use1[num] > 0) {
+            result++;
+        }
+    }
+    return result;
+}
 
 #if 0
 void stm_wait_for_current_inevitable_transaction(void)
@@ -202,7 +212,6 @@
 }
 #endif
 
-
 static void acquire_thread_segment(stm_thread_local_t *tl)
 {
     /* This function acquires a segment for the currently running thread,
diff --git a/c8/stm/sync.h b/c8/stm/sync.h
--- a/c8/stm/sync.h
+++ b/c8/stm/sync.h
@@ -22,6 +22,7 @@
 static void set_gs_register(char *value);
 static void ensure_gs_register(long segnum);
 
+static uint8_t number_of_segments_in_use(void);
 
 /* acquire and release one of the segments for running the given thread
    (must have the mutex acquired!) */

From pypy.commits at gmail.com  Tue May  2 10:34:27 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:27 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Make sequential
 transactions inevitable
Message-ID: <590898f3.8331c80a.ce486.5923@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2047:ffbfa75ef055
Date: 2017-04-29 13:56 +0200
http://bitbucket.org/pypy/stmgc/changeset/ffbfa75ef055/

Log:	Make sequential transactions inevitable

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1166,6 +1166,9 @@
     }
     _do_start_transaction(tl);
 
+    if (number_of_segments_in_use() < 2) {
+        stm_become_inevitable(tl, "single thread mode");
+    }
     if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
                                  in abort_data_structures_from_segment_num() */
         stm_update_transaction_length();

From pypy.commits at gmail.com  Tue May  2 10:34:30 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:30 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Fix name clash in macro
 expansion
Message-ID: <590898f6.5919370a.c8e6a.a644@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2048:78f3434d68d5
Date: 2017-04-29 16:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/78f3434d68d5/

Log:	Fix name clash in macro expansion

diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -27,9 +27,9 @@
 #define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop);            \
                       get_duration()
 
-#define stm_duration_payload(duration)                                      \
+#define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &duration };                                          \
+        { .duration = &duration_data };                                     \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 

From pypy.commits at gmail.com  Tue May  2 10:34:33 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:33 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Implement logging of
 adaptive mode
Message-ID: <590898f9.0603c80a.a9ee4.465a@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2049:68dab8f6465e
Date: 2017-04-29 16:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/68dab8f6465e/

Log:	Implement logging of adaptive mode

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -28,6 +28,16 @@
         LARGE_FILL_MARK_NURSERY_BYTES - DEFAULT_FILL_MARK_NURSERY_BYTES;
     stm_fill_mark_nursery_bytes =
         LARGE_FILL_MARK_NURSERY_BYTES - (relative_conflicts * max_reduction);
+    if (timing_enabled()) {
+        struct timespec relative_length = {
+            .tv_sec = (int)relative_conflicts,
+            .tv_nsec = (int)(fmod(relative_conflicts, 1) * 1000000000),
+        };
+        stm_duration_payload(relative_length);
+        stmcb_timing_event(
+            STM_SEGMENT->running_thread,
+            STM_SINGLE_THREAD_MODE_ADAPTIVE,
+            &stm_duration_payload);
 }
 
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -585,6 +585,8 @@
     STM_DURATION_MAJOR_GC_LOG_ONLY,
     STM_DURATION_MAJOR_GC_FULL,
 
+    STM_SINGLE_THREAD_MODE_ADAPTIVE,
+
     _STM_EVENT_N
 };
 

From pypy.commits at gmail.com  Tue May  2 10:34:36 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:36 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Merge all single thread
 mode enum values for consistency with analysis script
Message-ID: <590898fc.241f370a.5f673.e32a@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2050:784040daca87
Date: 2017-04-29 16:31 +0200
http://bitbucket.org/pypy/stmgc/changeset/784040daca87/

Log:	Merge all single thread mode enum values for consistency with
	analysis script

diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -585,6 +585,8 @@
     STM_DURATION_MAJOR_GC_LOG_ONLY,
     STM_DURATION_MAJOR_GC_FULL,
 
+    STM_SINGLE_THREAD_MODE_ON,
+    STM_SINGLE_THREAD_MODE_OFF,
     STM_SINGLE_THREAD_MODE_ADAPTIVE,
 
     _STM_EVENT_N

From pypy.commits at gmail.com  Tue May  2 10:34:38 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:38 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length: Implement exponential
 build up and back off in transaction length
Message-ID: <590898fe.c721ed0a.88e13.66c2@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length
Changeset: r2051:3ffb90144a7f
Date: 2017-05-01 18:08 +0200
http://bitbucket.org/pypy/stmgc/changeset/3ffb90144a7f/

Log:	Implement exponential build up and back off in transaction length

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -289,11 +289,6 @@
                         /* conflict! */
                         dprintf(("_stm_validate() failed for obj %p\n", obj));
 
-                        /* disregard race conditions */
-                        if (stm_global_conflicts < stm_max_conflicts) {
-                            stm_global_conflicts += 1;
-                        }
-
                         /* first reset all modified objects from the backup
                            copies as soon as the first conflict is detected;
                            then we will proceed below to update our segment
@@ -352,6 +347,7 @@
     }
 
     if (thread_local_for_logging != NULL) {
+        stm_transaction_length_handle_validation(thread_local_for_logging, needs_abort);
         stop_timer_and_publish_for_thread(
             thread_local_for_logging, STM_DURATION_VALIDATION);
     }
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -4,6 +4,7 @@
 #endif
 
 #include "finalizer.h"
+#include <math.h>
 
 /************************************************************/
 
@@ -14,30 +15,62 @@
 static uintptr_t _stm_nursery_start;
 
 #define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4)
-#define LARGE_FILL_MARK_NURSERY_BYTES   0x3000000000000000L;
+// just double the size at max
+// #define LARGE_FILL_MARK_NURSERY_BYTES   DEFAULT_FILL_MARK_NURSERY_BYTES
+#define LARGE_FILL_MARK_NURSERY_BYTES   0x10000000000L
+// #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
 
-// uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
-uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
+uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
+// uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-static uint32_t stm_max_conflicts = 1000;
-static uint32_t stm_global_conflicts = 0;
+static float stm_default_relative_transaction_length = 0.000001;
+static float *stm_relative_transaction_length_pointer =
+                                    &stm_default_relative_transaction_length;
+
+static float get_new_transaction_length(bool aborts, float previous) {
+    float new = previous;
+    if (aborts) {
+        if (previous > 0.000001) {
+            new = previous / 2;
+        } else if (previous > 0) {
+            new = 0;
+        }
+    } else {
+        if (previous - 0.0000001 < 0) {
+            new = 0.000001;
+        } else if (previous < 1) {
+            new = previous * 2;
+        }
+    }
+    return new;
+}
+
+static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) {
+    float actual = *stm_relative_transaction_length_pointer;
+    float expected;
+    do {
+        expected = actual;
+        float new = get_new_transaction_length(aborts, actual);
+        __atomic_exchange(
+            stm_relative_transaction_length_pointer, &new, &actual, __ATOMIC_RELAXED);
+    } while (fabs(actual - expected) > 0.0000001);
+}
 
 static void stm_update_transaction_length(void) {
-    float relative_conflicts = (float) stm_global_conflicts / stm_max_conflicts;
-    uintptr_t max_reduction =
-        LARGE_FILL_MARK_NURSERY_BYTES - DEFAULT_FILL_MARK_NURSERY_BYTES;
-    stm_fill_mark_nursery_bytes =
-        LARGE_FILL_MARK_NURSERY_BYTES - (relative_conflicts * max_reduction);
+    float relative_additional_length = *stm_relative_transaction_length_pointer;
+    stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES +
+        (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
     if (timing_enabled()) {
         struct timespec relative_length = {
-            .tv_sec = (int)relative_conflicts,
-            .tv_nsec = (int)(fmod(relative_conflicts, 1) * 1000000000),
+            .tv_sec = (int)relative_additional_length,
+            .tv_nsec = (int)(fmod(relative_additional_length, 1) * 1000000000),
         };
         stm_duration_payload(relative_length);
         stmcb_timing_event(
             STM_SEGMENT->running_thread,
             STM_SINGLE_THREAD_MODE_ADAPTIVE,
             &stm_duration_payload);
+    }
 }
 
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -59,6 +59,7 @@
 static uint32_t stm_max_conflicts;
 static uint32_t stm_global_conflicts;
 
+static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts);
 static void stm_update_transaction_length(void);
 
 #endif

From pypy.commits at gmail.com  Tue May  2 10:34:45 2017
From: pypy.commits at gmail.com (tobweber)
Date: Tue, 02 May 2017 07:34:45 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Merge fix for
 name clash in macro expansion
Message-ID: <59089905.a633ed0a.966f3.31c6@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-overheads-instrumentation
Changeset: r2052:4a93488e1444
Date: 2017-04-29 16:34 +0200
http://bitbucket.org/pypy/stmgc/changeset/4a93488e1444/

Log:	Merge fix for name clash in macro expansion

diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -27,9 +27,9 @@
 #define pause_timer() clock_gettime(CLOCK_MONOTONIC_RAW, &stop);            \
                       get_duration()
 
-#define stm_duration_payload(duration)                                      \
+#define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &duration };                                          \
+        { .duration = &duration_data };                                     \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 

From pypy.commits at gmail.com  Tue May  2 11:40:47 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 08:40:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add support for rejecting lone
 surrogates in utf16 and utf32 decoders.
Message-ID: <5908a87f.ee24ed0a.5cb17.7417@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91163:86c13077b36b
Date: 2017-04-19 00:53 +0200
http://bitbucket.org/pypy/pypy/changeset/86c13077b36b/

Log:	Add support for rejecting lone surrogates in utf16 and utf32
	decoders. (grafted from 7ce52a9f8d1f49eb4986b9bba8c8561060d49346)

diff --git a/rpython/rlib/runicode.py b/rpython/rlib/runicode.py
--- a/rpython/rlib/runicode.py
+++ b/rpython/rlib/runicode.py
@@ -591,7 +591,10 @@
 
 def unicode_encode_utf_16_helper(s, size, errors,
                                  errorhandler=None,
+                                 allow_surrogates=True,
                                  byteorder='little'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
     if size == 0:
         if byteorder == 'native':
             result = StringBuilder(2)
@@ -604,34 +607,60 @@
         _STORECHAR(result, 0xFEFF, BYTEORDER)
         byteorder = BYTEORDER
 
-    i = 0
-    while i < size:
-        ch = ord(s[i])
-        i += 1
-        ch2 = 0
-        if ch >= 0x10000:
-            ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF)
-            ch  = 0xD800 | ((ch-0x10000) >> 10)
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
 
-        _STORECHAR(result, ch, byteorder)
-        if ch2:
-            _STORECHAR(result, ch2, byteorder)
+        if ch < 0xD800:
+            _STORECHAR(result, ch, byteorder)
+        elif ch >= 0x10000:
+            _STORECHAR(result, 0xD800 | ((ch-0x10000) >> 10), byteorder)
+            _STORECHAR(result, 0xDC00 | ((ch-0x10000) & 0x3FF), byteorder)
+        elif ch >= 0xE000 or allow_surrogates:
+            _STORECHAR(result, ch, byteorder)
+        else:
+            ru, rs, pos = errorhandler(errors, 'utf16',
+                                       'surrogates not allowed',
+                                       s, pos-1, pos)
+            if rs is not None:
+                # py3k only
+                if len(rs) % 2 != 0:
+                    errorhandler('strict', 'utf16',
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+                result.append(rs)
+                continue
+            for ch in ru:
+                if ord(ch) < 0xD800:
+                    _STORECHAR(result, ord(ch), byteorder)
+                else:
+                    errorhandler('strict', 'utf16',
+                                 'surrogates not allowed',
+                                 s, pos-1, pos)
+            continue
 
     return result.build()
 
 def unicode_encode_utf_16(s, size, errors,
-                          errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "native")
+                          errorhandler=None,
+                          allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native")
 
 
 def unicode_encode_utf_16_be(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "big")
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big")
 
 
 def unicode_encode_utf_16_le(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_16_helper(s, size, errors, errorhandler, "little")
+                             errorhandler=None,
+                             allow_surrogates=True):
+    return unicode_encode_utf_16_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little")
 
 
 # ____________________________________________________________
@@ -756,7 +785,10 @@
 
 def unicode_encode_utf_32_helper(s, size, errors,
                                  errorhandler=None,
+                                 allow_surrogates=True,
                                  byteorder='little'):
+    if errorhandler is None:
+        errorhandler = default_unicode_error_encode
     if size == 0:
         if byteorder == 'native':
             result = StringBuilder(4)
@@ -769,33 +801,57 @@
         _STORECHAR32(result, 0xFEFF, BYTEORDER)
         byteorder = BYTEORDER
 
-    i = 0
-    while i < size:
-        ch = ord(s[i])
-        i += 1
+    pos = 0
+    while pos < size:
+        ch = ord(s[pos])
+        pos += 1
         ch2 = 0
-        if MAXUNICODE < 65536 and 0xD800 <= ch <= 0xDBFF and i < size:
-            ch2 = ord(s[i])
-            if 0xDC00 <= ch2 <= 0xDFFF:
-                ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
-                i += 1
+        if 0xD800 <= ch < 0xDC00:
+            if not allow_surrogates:
+                ru, rs, pos = errorhandler(errors, 'utf32',
+                                           'surrogates not allowed',
+                                           s, pos-1, pos)
+                if rs is not None:
+                    # py3k only
+                    if len(rs) % 4 != 0:
+                        errorhandler('strict', 'utf32',
+                                     'surrogates not allowed',
+                                     s, pos-1, pos)
+                    result.append(rs)
+                    continue
+                for ch in ru:
+                    if ord(ch) < 0xD800:
+                        _STORECHAR32(result, ord(ch), byteorder)
+                    else:
+                        errorhandler('strict', 'utf32',
+                                     'surrogates not allowed',
+                                     s, pos-1, pos)
+                continue
+            elif MAXUNICODE < 65536 and pos < size:
+                ch2 = ord(s[pos])
+                if 0xDC00 <= ch2 < 0xE000:
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+                    pos += 1
         _STORECHAR32(result, ch, byteorder)
 
     return result.build()
 
 def unicode_encode_utf_32(s, size, errors,
-                          errorhandler=None):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler, "native")
+                          errorhandler=None, allow_surrogates=True):
+    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "native")
 
 
 def unicode_encode_utf_32_be(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler, "big")
+                             errorhandler=None, allow_surrogates=True):
+    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "big")
 
 
 def unicode_encode_utf_32_le(s, size, errors,
-                             errorhandler=None):
-    return unicode_encode_utf_32_helper(s, size, errors, errorhandler, "little")
+                             errorhandler=None, allow_surrogates=True):
+    return unicode_encode_utf_32_helper(s, size, errors, errorhandler,
+                                        allow_surrogates, "little")
 
 
 # ____________________________________________________________
diff --git a/rpython/rlib/test/test_runicode.py b/rpython/rlib/test/test_runicode.py
--- a/rpython/rlib/test/test_runicode.py
+++ b/rpython/rlib/test/test_runicode.py
@@ -223,6 +223,40 @@
         py.test.raises(UnicodeDecodeError, runicode.str_decode_utf_16_le,
                        s, len(s), True)
 
+    def test_utf16_surrogates(self):
+        assert runicode.unicode_encode_utf_16_be(
+            u"\ud800", 1, None) == '\xd8\x00'
+        py.test.raises(UnicodeEncodeError, runicode.unicode_encode_utf_16_be,
+                       u"\ud800", 1, None, allow_surrogates=False)
+        def replace_with(ru, rs):
+            def errorhandler(errors, enc, msg, u, startingpos, endingpos):
+                if errors == 'strict':
+                    raise UnicodeEncodeError(enc, u, startingpos,
+                                             endingpos, msg)
+                return ru, rs, endingpos
+            return runicode.unicode_encode_utf_16_be(
+                u"<\ud800>", 3, None,
+                errorhandler, allow_surrogates=False)
+        assert replace_with(u'rep', None) == '\x00<\x00r\x00e\x00p\x00>'
+        assert replace_with(None, '\xca\xfe') == '\x00<\xca\xfe\x00>'
+
+    def test_utf32_surrogates(self):
+        assert runicode.unicode_encode_utf_32_be(
+            u"\ud800", 1, None) == '\x00\x00\xd8\x00'
+        py.test.raises(UnicodeEncodeError, runicode.unicode_encode_utf_32_be,
+                       u"\ud800", 1, None, allow_surrogates=False)
+        def replace_with(ru, rs):
+            def errorhandler(errors, enc, msg, u, startingpos, endingpos):
+                if errors == 'strict':
+                    raise UnicodeEncodeError(enc, u, startingpos,
+                                             endingpos, msg)
+                return ru, rs, endingpos
+            return runicode.unicode_encode_utf_32_be(
+                u"<\ud800>", 3, None,
+                errorhandler, allow_surrogates=False)
+        assert replace_with(u'rep', None) == u'<rep>'.encode('utf-32-be')
+        assert replace_with(None, '\xca\xfe\xca\xfe') == '\x00\x00\x00<\xca\xfe\xca\xfe\x00\x00\x00>'
+
     def test_utf7_bugs(self):
         u = u'A\u2262\u0391.'
         assert runicode.unicode_encode_utf_7(u, len(u), None) == 'A+ImIDkQ.'

From pypy.commits at gmail.com  Tue May  2 11:51:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 08:51:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Attempt to fix build_cffi_imports on
 Windows: args should be passed as unicode strings.
Message-ID: <5908aafe.1327370a.7cd8c.74cf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91164:fc8f35c963af
Date: 2017-04-02 11:57 +0200
http://bitbucket.org/pypy/pypy/changeset/fc8f35c963af/

Log:	Attempt to fix build_cffi_imports on Windows: args should be passed
	as unicode strings. (grafted from
	541dbb06046f547b72418bd6966627e3848e953d)

diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -15,7 +15,7 @@
     text = str
 
 def run_subprocess(executable, args, env=None, cwd=None):
-    if isinstance(args, list):
+    if isinstance(args, list) and sys.platform != 'win32':
         args = [a.encode('latin1') if isinstance(a, text) else a
                 for a in args]
     return _run(executable, args, env, cwd)

From pypy.commits at gmail.com  Tue May  2 11:51:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 08:51:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Popen.communicate() returns bytes if no
 encoding was specified.
Message-ID: <5908ab02.fa29c80a.a9f0.809c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91165:5218685456b2
Date: 2017-04-02 17:09 +0200
http://bitbucket.org/pypy/pypy/changeset/5218685456b2/

Log:	Popen.communicate() returns bytes if no encoding was specified.
	(grafted from 1fb74e1664446090361a9fe4ec11a67740e13d2d)

diff --git a/rpython/tool/runsubprocess.py b/rpython/tool/runsubprocess.py
--- a/rpython/tool/runsubprocess.py
+++ b/rpython/tool/runsubprocess.py
@@ -49,7 +49,7 @@
     pipe = Popen(args, stdout=PIPE, stderr=PIPE, shell=shell, env=env, cwd=cwd)
     stdout, stderr = pipe.communicate()
     if (sys.platform == 'win32' and pipe.returncode == 1 and 
-        'is not recognized' in stderr):
+        b'is not recognized' in stderr):
         # Setting shell=True on windows messes up expected exceptions
         raise EnvironmentError(stderr)
     return pipe.returncode, stdout, stderr

From pypy.commits at gmail.com  Tue May  2 11:58:37 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 08:58:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Attempt to import interp_scandir on
 Windows
Message-ID: <5908acad.0a39ed0a.6bc7c.3453@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91166:5d9621822874
Date: 2017-04-01 18:42 +0200
http://bitbucket.org/pypy/pypy/changeset/5d9621822874/

Log:	Attempt to import interp_scandir on Windows (grafted from
	ba2d0d7ea48e2dfafed9c17d9a125a46664247bb)

diff --git a/rpython/rlib/rposix_scandir.py b/rpython/rlib/rposix_scandir.py
--- a/rpython/rlib/rposix_scandir.py
+++ b/rpython/rlib/rposix_scandir.py
@@ -1,4 +1,4 @@
-from rpython.rlib import rposix
+from rpython.rlib import rposix, rwin32
 from rpython.rlib.objectmodel import specialize
 from rpython.rtyper.lltypesystem import lltype, rffi
 
@@ -17,7 +17,8 @@
 def closedir(dirp):
     rposix.c_closedir(dirp)
 
-NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
+if not rwin32.WIN32:
+    NULL_DIRP = lltype.nullptr(rposix.DIRP.TO)
 
 def nextentry(dirp):
     """Read the next entry and returns an opaque object.

From pypy.commits at gmail.com  Tue May  2 12:10:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 09:10:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: blindly change this from -1 to an
 arbitrary large number on 32 bit systems
Message-ID: <5908af8b.4433370a.b31ab.7bf5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91167:5f9e5afded55
Date: 2017-03-21 22:22 +0200
http://bitbucket.org/pypy/pypy/changeset/5f9e5afded55/

Log:	blindly change this from -1 to an arbitrary large number on 32 bit
	systems (grafted from eecdab8e95b5c78071de41b3a5b0007c5d994f4b)

diff --git a/rpython/rlib/rsre/rsre_char.py b/rpython/rlib/rsre/rsre_char.py
--- a/rpython/rlib/rsre/rsre_char.py
+++ b/rpython/rlib/rsre/rsre_char.py
@@ -28,7 +28,7 @@
     MAXGROUPS = int(2**31 - 1)
 else:
     MAXREPEAT = int(2**31 - 1)
-    MAXGROUPS = int((2**31 / sys.maxint / 2) - 1)
+    MAXGROUPS = int(2**30 - 1)
 
 # In _sre.c this is bytesize of the code word type of the C implementation.
 # There it's 2 for normal Python builds and more for wide unicode builds (large

From pypy.commits at gmail.com  Tue May  2 12:15:56 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 09:15:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport changes from py3.5 branch
Message-ID: <5908b0bc.01b9370a.fb589.9a54@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91168:0c21123991d2
Date: 2017-05-02 17:15 +0100
http://bitbucket.org/pypy/pypy/changeset/0c21123991d2/

Log:	Backport changes from py3.5 branch

diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -6,8 +6,10 @@
 #define SINGLE_BUF_SIZE (8192 - 2 * sizeof(unsigned int))
 
 #ifdef VMPROF_WINDOWS
-#include "msiinttypes/inttypes.h"
-#include "msiinttypes/stdint.h"
+#include <crtdefs.h>
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+typedef intptr_t ssize_t;
 #else
 #include <inttypes.h>
 #include <stdint.h>

From pypy.commits at gmail.com  Tue May  2 12:48:02 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 09:48:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport rposix changes from py3.5
 branch
Message-ID: <5908b842.fd33c80a.27813.791b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91169:02051354da6b
Date: 2017-05-02 17:47 +0100
http://bitbucket.org/pypy/pypy/changeset/02051354da6b/

Log:	Backport rposix changes from py3.5 branch

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -261,6 +261,8 @@
 
     HAVE_UTIMES = rffi_platform.Has('utimes')
     HAVE_D_TYPE = rffi_platform.Has('DT_UNKNOWN')
+    HAVE_FALLOCATE = rffi_platform.Has('posix_fallocate')
+    HAVE_FADVISE = rffi_platform.Has('posix_fadvise')
     UTIMBUF = rffi_platform.Struct('struct %sutimbuf' % UNDERSCORE_ON_WIN32,
                                    [('actime', rffi.INT),
                                     ('modtime', rffi.INT)])
@@ -449,7 +451,7 @@
 def close(fd):
     validate_fd(fd)
     handle_posix_error('close', c_close(fd))
-
+    
 c_lseek = external('_lseeki64' if _WIN32 else 'lseek',
                    [rffi.INT, rffi.LONGLONG, rffi.INT], rffi.LONGLONG,
                    macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO)
@@ -490,6 +492,41 @@
         with rffi.scoped_nonmovingbuffer(data) as buf:
             return handle_posix_error('pwrite', c_pwrite(fd, buf, count, offset))
 
+    if HAVE_FALLOCATE:
+        c_posix_fallocate = external('posix_fallocate',
+                                     [rffi.INT, OFF_T, OFF_T], rffi.INT,
+                                     save_err=rffi.RFFI_SAVE_ERRNO)
+
+        @enforceargs(int, None, None)
+        def posix_fallocate(fd, offset, length):
+            validate_fd(fd)
+            return handle_posix_error('posix_fallocate', c_posix_fallocate(fd, offset, length))
+
+    if HAVE_FADVISE:
+        class CConfig:
+            _compilation_info_ = eci
+            POSIX_FADV_WILLNEED = rffi_platform.DefinedConstantInteger('POSIX_FADV_WILLNEED')
+            POSIX_FADV_NORMAL = rffi_platform.DefinedConstantInteger('POSIX_FADV_NORMAL')
+            POSIX_FADV_SEQUENTIAL = rffi_platform.DefinedConstantInteger('POSIX_FADV_SEQUENTIAL')
+            POSIX_FADV_RANDOM= rffi_platform.DefinedConstantInteger('POSIX_FADV_RANDOM')
+            POSIX_FADV_NOREUSE = rffi_platform.DefinedConstantInteger('POSIX_FADV_NOREUSE')
+            POSIX_FADV_DONTNEED = rffi_platform.DefinedConstantInteger('POSIX_FADV_DONTNEED')
+
+        config = rffi_platform.configure(CConfig)
+        globals().update(config)
+
+        c_posix_fadvise = external('posix_fadvise',
+                                   [rffi.INT, OFF_T, OFF_T, rffi.INT], rffi.INT,
+                                   save_err=rffi.RFFI_SAVE_ERRNO)
+
+        @enforceargs(int, None, None, int)
+        def posix_fadvise(fd, offset, length, advice):
+            validate_fd(fd)
+            error = c_posix_fadvise(fd, offset, length, advice)
+            error = widen(error)
+            if error != 0:
+                raise OSError(error, 'posix_fadvise failed')
+
 c_ftruncate = external('ftruncate', [rffi.INT, rffi.LONGLONG], rffi.INT,
                        macro=_MACRO_ON_POSIX, save_err=rffi.RFFI_SAVE_ERRNO)
 c_fsync = external('fsync' if not _WIN32 else '_commit', [rffi.INT], rffi.INT,
@@ -2020,8 +2057,10 @@
         raise OSError(get_saved_errno(), "execve failed")
 
 if HAVE_LINKAT:
-    c_linkat = external('linkat',
-        [rffi.INT, rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT], rffi.INT)
+    c_linkat = external(
+        'linkat',
+        [rffi.INT, rffi.CCHARP, rffi.INT, rffi.CCHARP, rffi.INT], rffi.INT,
+        save_err=rffi.RFFI_SAVE_ERRNO)
 
     def linkat(src, dst, src_dir_fd=AT_FDCWD, dst_dir_fd=AT_FDCWD,
             follow_symlinks=True):
@@ -2036,7 +2075,8 @@
         handle_posix_error('linkat', error)
 
 if HAVE_FUTIMENS:
-    c_futimens = external('futimens', [rffi.INT, TIMESPEC2P], rffi.INT)
+    c_futimens = external('futimens', [rffi.INT, TIMESPEC2P], rffi.INT,
+                          save_err=rffi.RFFI_SAVE_ERRNO)
 
     def futimens(fd, atime, atime_ns, mtime, mtime_ns):
         l_times = lltype.malloc(TIMESPEC2P.TO, 2, flavor='raw')
@@ -2049,8 +2089,10 @@
         handle_posix_error('futimens', error)
 
 if HAVE_UTIMENSAT:
-    c_utimensat = external('utimensat',
-        [rffi.INT, rffi.CCHARP, TIMESPEC2P, rffi.INT], rffi.INT)
+    c_utimensat = external(
+        'utimensat',
+        [rffi.INT, rffi.CCHARP, TIMESPEC2P, rffi.INT], rffi.INT,
+        save_err=rffi.RFFI_SAVE_ERRNO)
 
     def utimensat(pathname, atime, atime_ns, mtime, mtime_ns,
             dir_fd=AT_FDCWD, follow_symlinks=True):
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -750,3 +750,34 @@
     finally:
         os.close(fd)
     py.test.raises(OSError, rposix.pwrite, fd, b'ea', 1)
+
+ at rposix_requires('posix_fadvise')
+def test_posix_fadvise():
+    fname = str(udir.join('test_os_posix_fadvise'))
+    fd = os.open(fname, os.O_CREAT | os.O_RDWR)
+    try:
+        os.write(fd, b"foobar")
+        assert rposix.posix_fadvise(fd, 0, 1, rposix.POSIX_FADV_WILLNEED) is None
+        assert rposix.posix_fadvise(fd, 1, 1, rposix.POSIX_FADV_NORMAL) is None
+        assert rposix.posix_fadvise(fd, 2, 1, rposix.POSIX_FADV_SEQUENTIAL) is None
+        assert rposix.posix_fadvise(fd, 3, 1, rposix.POSIX_FADV_RANDOM) is None
+        assert rposix.posix_fadvise(fd, 4, 1, rposix.POSIX_FADV_NOREUSE) is None
+        assert rposix.posix_fadvise(fd, 5, 1, rposix.POSIX_FADV_DONTNEED) is None
+        py.test.raises(OSError, rposix.posix_fadvise, fd, 6, 1, 1234567)
+    finally:
+        os.close(fd)
+
+ at rposix_requires('posix_fallocate')
+def test_posix_fallocate():
+    fname = str(udir.join('os_test.txt'))
+    fd = os.open(fname, os.O_WRONLY | os.O_CREAT, 0777)
+    try:
+        assert rposix.posix_fallocate(fd, 0, 10) == 0
+    except OSError as inst:
+        """ ZFS seems not to support fallocate.
+        so skipping solaris-based since it is likely to come with ZFS
+        """
+        if inst.errno != errno.EINVAL or not sys.platform.startswith("sunos"):
+            raise
+    finally:
+        os.close(fd)

From pypy.commits at gmail.com  Tue May  2 14:02:43 2017
From: pypy.commits at gmail.com (stefanor)
Date: Tue, 02 May 2017 11:02:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update includes README to mention new
 generated files, and the cpyext parse directory.
Message-ID: <5908c9c3.4386370a.7d5a5.63d8@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r91170:54dc01ff643b
Date: 2017-05-02 11:00 -0700
http://bitbucket.org/pypy/pypy/changeset/54dc01ff643b/

Log:	Update includes README to mention new generated files, and the
	cpyext parse directory.

	Brought to attention by issue #2550.

diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h

From pypy.commits at gmail.com  Tue May  2 14:46:59 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 11:46:59 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Backport rpython.rlib.buffer
 changes from py3.5
Message-ID: <5908d423.11a2370a.66d55.8107@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91171:4ec9314365c8
Date: 2017-05-02 18:02 +0100
http://bitbucket.org/pypy/pypy/changeset/4ec9314365c8/

Log:	Backport rpython.rlib.buffer changes from py3.5

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -1,13 +1,13 @@
 """
 Buffer protocol support.
 """
-from rpython.rlib import jit
-from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-        nonmoving_raw_ptr_for_resizable_list)
-
+from rpython.rlib.rgc import (
+    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
+from rpython.rlib.signature import signature
+from rpython.rlib import types
 
 class Buffer(object):
-    """Abstract base class for buffers."""
+    """Base class for buffers of bytes"""
     _attrs_ = ['readonly']
     _immutable_ = True
 
@@ -27,8 +27,8 @@
 
     def as_str_and_offset_maybe(self):
         """
-        If the buffer is backed by a string, return a pair (string, offset), where
-        offset is the offset inside the string where the buffer start.
+        If the buffer is backed by a string, return a pair (string, offset),
+        where offset is the offset inside the string where the buffer start.
         Else, return (None, 0).
         """
         return None, 0
@@ -44,6 +44,7 @@
         # May be overridden.  No bounds checks.
         return ''.join([self.getitem(i) for i in range(start, stop, step)])
 
+    @signature(types.any(), types.int(), types.int(), returns=types.str())
     def __getslice__(self, start, stop):
         return self.getslice(start, stop, 1, stop - start)
 
@@ -59,26 +60,24 @@
         for i in range(len(string)):
             self.setitem(start + i, string[i])
 
+class ByteBuffer(Buffer):
+    _immutable_ = True
+
+    def __init__(self, n):
+        self.data = resizable_list_supporting_raw_ptr(['\0'] * n)
+        self.readonly = False
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitem(self, index):
+        return self.data[index]
+
+    def setitem(self, index, char):
+        self.data[index] = char
+
     def get_raw_address(self):
-        raise ValueError("no raw buffer")
-
-    def getformat(self):
-        return 'B'
-
-    def getitemsize(self):
-        return 1
-
-    def getndim(self):
-        return 1
-
-    def getshape(self):
-        return [self.getlength()]
-
-    def getstrides(self):
-        return [1]
-
-    def releasebuffer(self):
-        pass
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
@@ -119,6 +118,8 @@
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
     _immutable_ = True
 
+    @signature(types.any(), types.instance(Buffer), types.int(), types.int(),
+               returns=types.none())
     def __init__(self, buffer, offset, size):
         self.readonly = buffer.readonly
         if isinstance(buffer, SubBuffer):     # don't nest them
@@ -149,7 +150,7 @@
     def as_str_and_offset_maybe(self):
         string, offset = self.buffer.as_str_and_offset_maybe()
         if string is not None:
-            return string, offset+self.offset
+            return string, offset + self.offset
         return None, 0
 
     def getitem(self, index):
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -8,7 +8,6 @@
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import newlist_hint, resizelist_hint, specialize, not_rpython
 from rpython.rlib.rarithmetic import ovfcheck, LONG_BIT as BLOOM_WIDTH
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb
 from rpython.rtyper.extregistry import ExtRegistryEntry
 from rpython.tool.pairtype import pairtype

From pypy.commits at gmail.com  Tue May  2 14:47:02 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 02 May 2017 11:47:02 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Backport memoryview changes
Message-ID: <5908d426.a82b370a.2088f.6486@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91172:fea54844618f
Date: 2017-05-02 19:45 +0100
http://bitbucket.org/pypy/pypy/changeset/fea54844618f/

Log:	Backport memoryview changes

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,321 @@
+from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.buffer import StringBuffer, SubBuffer
+
+from pypy.interpreter.error import oefmt
+
+class BufferInterfaceNotFound(Exception):
+    pass
+
+
+class BufferView(object):
+    """Abstract base class for buffers."""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        return ''.join(self._copy_buffer())
+
+    def getbytes(self, start, size):
+        """Return `size` bytes starting at byte offset `start`.
+
+        This is a low-level operation, it is up to the caller to ensure that
+        the data requested actually correspond to items accessible from the
+        BufferView.
+        Note that `start` may be negative, e.g. if the buffer is reversed.
+        """
+        raise NotImplementedError
+
+    def setbytes(self, start, string):
+        raise NotImplementedError
+
+    def get_raw_address(self):
+        raise ValueError("no raw buffer")
+
+    def as_readbuf(self):
+        # Inefficient. May be overridden.
+        return StringBuffer(self.as_str())
+
+    def as_writebuf(self):
+        """Return a writable Buffer sharing the same data as `self`."""
+        raise BufferInterfaceNotFound
+
+    def getformat(self):
+        raise NotImplementedError
+
+    def getitemsize(self):
+        raise NotImplementedError
+
+    def getndim(self):
+        raise NotImplementedError
+
+    def getshape(self):
+        raise NotImplementedError
+
+    def getstrides(self):
+        raise NotImplementedError
+
+    def releasebuffer(self):
+        pass
+
+    def value_from_bytes(self, space, s):
+        from pypy.module.struct.formatiterator import UnpackFormatIterator
+        buf = StringBuffer(s)
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(self.getformat())
+        return fmtiter.result_w[0]
+
+    def bytes_from_value(self, space, w_val):
+        from pypy.module.struct.formatiterator import PackFormatIterator
+        itemsize = self.getitemsize()
+        fmtiter = PackFormatIterator(space, [w_val], itemsize)
+        try:
+            fmtiter.interpret(self.getformat())
+        except StructError as e:
+            raise oefmt(space.w_TypeError,
+                        "memoryview: invalid type for format '%s'",
+                        self.getformat())
+        return fmtiter.result.build()
+
+    def _copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.getbytes(0, itemsize)]
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim + 1, data, off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        bytesize = self.getlength()
+        copiedbytes = 0
+        for i in range(step):
+            bytes = self.getbytes(off, itemsize)
+            data.append(bytes)
+            copiedbytes += len(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if copiedbytes >= bytesize:
+                break
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        shape = self.getshape()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        # TODO suboffsets?
+        strides = self.getstrides()
+        return strides[dim] * index
+
+    def w_getitem(self, space, idx):
+        offset = self.get_offset(space, 0, idx)
+        itemsize = self.getitemsize()
+        # TODO: this probably isn't very fast
+        data = self.getbytes(offset, itemsize)
+        return self.value_from_bytes(space, data)
+
+    def new_slice(self, start, step, slicelength):
+        return BufferSlice(self, start, step, slicelength)
+
+    def setitem_w(self, space, idx, w_obj):
+        offset = self.get_offset(space, 0, idx)
+        # TODO: this probably isn't very fast
+        byteval = self.bytes_from_value(space, w_obj)
+        self.setbytes(offset, byteval)
+
+    def w_tolist(self, space):
+        dim = self.getndim()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            n = self.getshape()[0]
+            values_w = [space.ord(self.w_getitem(space, i)) for i in range(n)]
+            return space.newlist(values_w)
+        else:
+            return self._tolist_rec(space, 0, 0)
+
+    def _tolist_rec(self, space, start, idim):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim + 1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            values_w = [
+                self.value_from_bytes(space, self.getbytes(pos, itemsize))
+                for pos in range(start, start + bytecount, stride)]
+            return space.newlist(values_w)
+
+        items = [None] * dimshape
+        for i in range(dimshape):
+            item = self._tolist_rec(space, start, idim + 1)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+    def wrap(self, space):
+        return space.newmemoryview(self)
+
+
+class SimpleView(BufferView):
+    _attrs_ = ['readonly', 'data']
+    _immutable_ = True
+
+    def __init__(self, data):
+        self.data = data
+        self.readonly = self.data.readonly
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def as_str(self):
+        return self.data.as_str()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def setbytes(self, offset, s):
+        self.data.setslice(offset, s)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_readbuf(self):
+        return self.data
+
+    def as_writebuf(self):
+        assert not self.data.readonly
+        return self.data
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        assert dim == 0
+        nitems = self.getlength()
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        return index
+
+    def w_getitem(self, space, idx):
+        idx = self.get_offset(space, 0, idx)
+        ch = self.data[idx]
+        return space.newbytes(ch)
+
+    def new_slice(self, start, step, slicelength):
+        if step == 1:
+            return SimpleView(SubBuffer(self.data, start, slicelength))
+        else:
+            return BufferSlice(self, start, step, slicelength)
+
+    def setitem_w(self, space, idx, w_obj):
+        idx = self.get_offset(space, 0, idx)
+        self.data[idx] = space.byte_w(w_obj)
+
+
+class BufferSlice(BufferView):
+    _immutable_ = True
+    _attrs_ = ['parent', 'readonly', 'shape', 'strides', 'start', 'step']
+
+    def __init__(self, parent, start, step, length):
+        self.parent = parent
+        self.readonly = self.parent.readonly
+        self.strides = parent.getstrides()[:]
+        self.start = start
+        self.step = step
+        self.strides[0] *= step
+        self.shape = parent.getshape()[:]
+        self.shape[0] = length
+
+    def getlength(self):
+        return self.shape[0] * self.getitemsize()
+
+    def getbytes(self, start, size):
+        offset = self.start * self.parent.getstrides()[0]
+        return self.parent.getbytes(offset + start, size)
+
+    def setbytes(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        offset = self.start * self.parent.getstrides()[0]
+        self.parent.setbytes(offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        offset = self.start * self.parent.getstrides()[0]
+        return rffi.ptradd(self.parent.get_raw_address(), offset)
+
+    def getformat(self):
+        return self.parent.getformat()
+
+    def getitemsize(self):
+        return self.parent.getitemsize()
+
+    def getndim(self):
+        return self.parent.getndim()
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides
+
+    def parent_index(self, idx):
+        return self.start + self.step * idx
+
+    def w_getitem(self, space, idx):
+        return self.parent.w_getitem(space, self.parent_index(idx))
+
+    def new_slice(self, start, step, slicelength):
+        real_start = start + self.start
+        real_step = self.step * step
+        return BufferSlice(self.parent, real_start, real_step, slicelength)
+
+    def setitem_w(self, space, idx, w_obj):
+        return self.parent.setitem_w(space, self.parent_index(idx), w_obj)
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
@@ -19,7 +20,7 @@
 
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf
+        return SimpleView(self.buf)
 
     def readbuf_w(self, space):
         return self.buf
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -8,6 +8,7 @@
 from rpython.rlib.rstring import StringBuilder, replace
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import (
     WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
@@ -455,7 +456,7 @@
 
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, True)
-        return StringBuffer(self._value)
+        return SimpleView(StringBuffer(self._value))
 
     def readbuf_w(self, space):
         return StringBuffer(self._value)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -3,48 +3,78 @@
 """
 import operator
 
-from rpython.rlib.buffer import Buffer, SubBuffer
+from rpython.rlib.buffer import SubBuffer
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app
 from pypy.interpreter.typedef import TypeDef, GetSetProperty
 
+MEMORYVIEW_MAX_DIM = 64
+MEMORYVIEW_SCALAR   = 0x0001
+MEMORYVIEW_C        = 0x0002
+MEMORYVIEW_FORTRAN  = 0x0004
+MEMORYVIEW_SCALAR   = 0x0008
+MEMORYVIEW_PIL      = 0x0010
+
 
 class W_MemoryView(W_Root):
     """Implement the built-in 'memoryview' type as a wrapper around
     an interp-level buffer.
     """
-    _attrs_ = ['buf']
 
-    def __init__(self, buf):
-        assert isinstance(buf, Buffer)
-        self.buf = buf
+    def __init__(self, view):
+        assert isinstance(view, BufferView)
+        self.view = view
+        self._hash = -1
+        self.flags = 0
+        self._init_flags()
+
+    def getndim(self):
+        return self.view.getndim()
+
+    def getshape(self):
+        return self.view.getshape()
+
+    def getstrides(self):
+        return self.view.getstrides()
+
+    def getitemsize(self):
+        return self.view.getitemsize()
+
+    def getformat(self):
+        return self.view.getformat()
 
     def buffer_w(self, space, flags):
-        space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf
+        self._check_released(space)
+        space.check_buf_flags(flags, self.view.readonly)
+        return self.view
 
     @staticmethod
     def descr_new_memoryview(space, w_subtype, w_object):
-        return W_MemoryView(space.buffer_w(w_object, space.BUF_FULL_RO))
+        if isinstance(w_object, W_MemoryView):
+            w_object._check_released(space)
+            return W_MemoryView.copy(w_object)
+        view = space.buffer_w(w_object, space.BUF_FULL_RO)
+        return view.wrap(space)
 
     def _make_descr__cmp(name):
         def descr__cmp(self, space, w_other):
             if isinstance(w_other, W_MemoryView):
                 # xxx not the most efficient implementation
-                str1 = self.as_str()
-                str2 = w_other.as_str()
+                str1 = self.view.as_str()
+                str2 = w_other.view.as_str()
                 return space.newbool(getattr(operator, name)(str1, str2))
 
             try:
-                buf = space.buffer_w(w_other, space.BUF_CONTIG_RO)
+                view = space.buffer_w(w_other, space.BUF_CONTIG_RO)
             except OperationError as e:
                 if not e.match(space, space.w_TypeError):
                     raise
                 return space.w_NotImplemented
             else:
-                str1 = self.as_str()
-                str2 = buf.as_str()
+                str1 = self.view.as_str()
+                str2 = view.as_str()
                 return space.newbool(getattr(operator, name)(str1, str2))
         descr__cmp.func_name = name
         return descr__cmp
@@ -60,101 +90,149 @@
         return self.buf.as_str()
 
     def getlength(self):
-        return self.buf.getlength()
+        return self.view.getlength()
 
     def descr_tobytes(self, space):
-        return space.newbytes(self.as_str())
+        self._check_released(space)
+        return space.newbytes(self.view.as_str())
 
     def descr_tolist(self, space):
-        buf = self.buf
-        result = []
-        for i in range(buf.getlength()):
-            result.append(space.newint(ord(buf.getitem(i))))
-        return space.newlist(result)
+        self._check_released(space)
+        return self.view.w_tolist(space)
+
+    def _decode_index(self, space, w_index, is_slice):
+        shape = self.getshape()
+        if len(shape) == 0:
+            count = 1
+        else:
+            count = shape[0]
+        return space.decode_index4(w_index, count)
 
     def descr_getitem(self, space, w_index):
-        start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
+        is_slice = space.isinstance_w(w_index, space.w_slice)
+        start, stop, step, slicelength = self._decode_index(space, w_index, is_slice)
+        # ^^^ for a non-slice index, this returns (index, 0, 0, 1)
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
         if step == 0:  # index only
-            return space.newbytes(self.buf.getitem(start))
+            dim = self.getndim()
+            if dim == 0:
+                raise oefmt(space.w_TypeError, "invalid indexing of 0-dim memory")
+            elif dim == 1:
+                return self.view.w_getitem(space, start)
+            else:
+                raise oefmt(space.w_NotImplementedError, "multi-dimensional sub-views are not implemented")
+        elif is_slice:
+            return self.view.new_slice(start, step, slicelength).wrap(space)
+        # multi index is handled at the top of this function
         else:
-            buf = SubBuffer(self.buf, start, size)
-            return W_MemoryView(buf)
+            raise TypeError("memoryview: invalid slice key")
 
     def descr_setitem(self, space, w_index, w_obj):
-        if self.buf.readonly:
+        self._check_released(space)
+        if self.view.readonly:
             raise oefmt(space.w_TypeError, "cannot modify read-only memory")
         start, stop, step, size = space.decode_index4(w_index, self.getlength())
-        itemsize = self.buf.getitemsize()
-        if itemsize > 1:
-            start *= itemsize
-            size *= itemsize
-            stop  = start + size
-            if step == 0:
-                step = 1
-            if stop > self.getlength():
-                raise oefmt(space.w_IndexError, 'index out of range')
         if step not in (0, 1):
             raise oefmt(space.w_NotImplementedError, "")
-        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-        if value.getlength() != size:
-            raise oefmt(space.w_ValueError,
-                        "cannot modify size of memoryview object")
+        is_slice = space.isinstance_w(w_index, space.w_slice)
+        start, stop, step, slicelength = self._decode_index(space, w_index, is_slice)
+        itemsize = self.getitemsize()
         if step == 0:  # index only
-            self.buf.setitem(start, value.getitem(0))
+            self.view.setitem_w(space, start, w_obj)
         elif step == 1:
-            self.buf.setslice(start, value.as_str())
+            value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            if value.getlength() != slicelength * itemsize:
+                raise oefmt(space.w_ValueError,
+                            "cannot modify size of memoryview object")
+            self.view.setbytes(start * itemsize, value.as_str())
 
     def descr_len(self, space):
-        return space.newint(self.buf.getlength() / self.buf.getitemsize())
+        self._check_released(space)
+        dim = self.getndim()
+        if dim == 0:
+            return space.newint(1)
+        shape = self.getshape()
+        return space.newint(shape[0])
 
     def w_get_format(self, space):
-        return space.newtext(self.buf.getformat())
+        self._check_released(space)
+        return space.newtext(self.getformat())
 
     def w_get_itemsize(self, space):
-        return space.newint(self.buf.getitemsize())
+        self._check_released(space)
+        return space.newint(self.getitemsize())
 
     def w_get_ndim(self, space):
-        return space.newint(self.buf.getndim())
+        self._check_released(space)
+        return space.newint(self.getndim())
 
     def w_is_readonly(self, space):
-        return space.newbool(bool(self.buf.readonly))
+        self._check_released(space)
+        return space.newbool(bool(self.view.readonly))
 
     def w_get_shape(self, space):
-        if self.buf.getndim() == 0:
+        self._check_released(space)
+        if self.view.getndim() == 0:
             return space.w_None
-        return space.newtuple([space.newint(x) for x in self.buf.getshape()])
+        return space.newtuple([space.newint(x) for x in self.getshape()])
 
     def w_get_strides(self, space):
-        if self.buf.getndim() == 0:
+        self._check_released(space)
+        if self.view.getndim() == 0:
             return space.w_None
-        return space.newtuple([space.newint(x) for x in self.buf.getstrides()])
+        return space.newtuple([space.newint(x) for x in self.getstrides()])
 
     def w_get_suboffsets(self, space):
+        self._check_released(space)
         # I've never seen anyone filling this field
         return space.w_None
 
+    def _check_released(self, space):
+        if self.view is None:
+            raise oefmt(space.w_ValueError,
+                        "operation forbidden on released memoryview object")
+
     def descr_pypy_raw_address(self, space):
         from rpython.rtyper.lltypesystem import lltype, rffi
         try:
-            ptr = self.buf.get_raw_address()
+            ptr = self.view.get_raw_address()
         except ValueError:
-            # report the error using the RPython-level internal repr of self.buf
+            # report the error using the RPython-level internal repr of
+            # self.view
             msg = ("cannot find the underlying address of buffer that "
-                   "is internally %r" % (self.buf,))
+                   "is internally %r" % (self.view,))
             raise OperationError(space.w_ValueError, space.newtext(msg))
         return space.newint(rffi.cast(lltype.Signed, ptr))
 
+    def _init_flags(self):
+        ndim = self.getndim()
+        flags = 0
+        if ndim == 0:
+            flags |= MEMORYVIEW_SCALAR | MEMORYVIEW_C | MEMORYVIEW_FORTRAN
+        elif ndim == 1:
+            shape = self.getshape()
+            strides = self.getstrides()
+            if shape[0] == 1 or strides[0] == self.getitemsize():
+                flags |= MEMORYVIEW_C | MEMORYVIEW_FORTRAN
+        else:
+            ndim = self.getndim()
+            shape = self.getshape()
+            strides = self.getstrides()
+            itemsize = self.getitemsize()
+            if PyBuffer_isContiguous(None, ndim, shape, strides,
+                                      itemsize, 'C'):
+                flags |= MEMORYVIEW_C
+            if PyBuffer_isContiguous(None, ndim, shape, strides,
+                                      itemsize, 'F'):
+                flags |= MEMORYVIEW_FORTRAN
+
+        if False:  # TODO missing suboffsets
+            flags |= MEMORYVIEW_PIL
+            flags &= ~(MEMORYVIEW_C|MEMORYVIEW_FORTRAN)
+
+        self.flags = flags
+
 W_MemoryView.typedef = TypeDef(
     "memoryview",
     __doc__ = """\
@@ -182,3 +260,49 @@
     _pypy_raw_address = interp2app(W_MemoryView.descr_pypy_raw_address),
     )
 W_MemoryView.typedef.acceptable_as_base_class = False
+
+def _IsFortranContiguous(ndim, shape, strides, itemsize):
+    if ndim == 0:
+        return 1
+    if not strides:
+        return ndim == 1
+    sd = itemsize
+    if ndim == 1:
+        return shape[0] == 1 or sd == strides[0]
+    for i in range(ndim):
+        dim = shape[i]
+        if dim == 0:
+            return 1
+        if strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def _IsCContiguous(ndim, shape, strides, itemsize):
+    if ndim == 0:
+        return 1
+    if not strides:
+        return ndim == 1
+    sd = itemsize
+    if ndim == 1:
+        return shape[0] == 1 or sd == strides[0]
+    for i in range(ndim - 1, -1, -1):
+        dim = shape[i]
+        if dim == 0:
+            return 1
+        if strides[i] != sd:
+            return 0
+        sd *= dim
+    return 1
+
+def PyBuffer_isContiguous(suboffsets, ndim, shape, strides, itemsize, fort):
+    if suboffsets:
+        return 0
+    if (fort == 'C'):
+        return _IsCContiguous(ndim, shape, strides, itemsize)
+    elif (fort == 'F'):
+        return _IsFortranContiguous(ndim, shape, strides, itemsize)
+    elif (fort == 'A'):
+        return (_IsCContiguous(ndim, shape, strides, itemsize) or
+                _IsFortranContiguous(ndim, shape, strides, itemsize))
+    return 0
\ No newline at end of file
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -341,6 +341,9 @@
         ret = W_Buffer(obj)
         return ret
 
+    def newmemoryview(self, view):
+        return W_MemoryView(view)
+
     def newbytes(self, s):
         assert isinstance(s, str)
         return W_BytesObject(s)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -1,13 +1,13 @@
 class AppTestMemoryView:
     def test_basic(self):
-        v = memoryview("abc")
-        assert v.tobytes() == "abc"
+        v = memoryview(b"abc")
+        assert v.tobytes() == b"abc"
         assert len(v) == 3
         assert list(v) == ['a', 'b', 'c']
         assert v.tolist() == [97, 98, 99]
         assert v[1] == "b"
         assert v[-1] == "c"
-        exc = raises(TypeError, "v[1] = 'x'")
+        exc = raises(TypeError, "v[1] = b'x'")
         assert str(exc.value) == "cannot modify read-only memory"
         assert v.readonly is True
         w = v[1:234]
@@ -17,7 +17,7 @@
         assert str(exc.value) == ""
 
     def test_rw(self):
-        data = bytearray('abcefg')
+        data = bytearray(b'abcefg')
         v = memoryview(data)
         assert v.readonly is False
         v[0] = 'z'
@@ -32,7 +32,7 @@
         assert str(exc.value) == ""
 
     def test_memoryview_attrs(self):
-        v = memoryview("a"*100)
+        v = memoryview(b"a"*100)
         assert v.format == "B"
         assert v.itemsize == 1
         assert v.shape == (100,)
@@ -40,22 +40,25 @@
         assert v.strides == (1,)
 
     def test_suboffsets(self):
-        v = memoryview("a"*100)
+        v = memoryview(b"a"*100)
         assert v.suboffsets == None
         v = memoryview(buffer("a"*100, 2))
         assert v.shape == (98,)
         assert v.suboffsets == None
 
     def test_compare(self):
-        assert memoryview("abc") == "abc"
-        assert memoryview("abc") == bytearray("abc")
-        assert memoryview("abc") != 3
+        assert memoryview(b"abc") == b"abc"
+        assert memoryview(b"abc") == bytearray(b"abc")
+        assert memoryview(b"abc") != 3
         assert not memoryview("abc") == u"abc"
         assert memoryview("abc") != u"abc"
         assert not u"abc" == memoryview("abc")
         assert u"abc" != memoryview("abc")
 
     def test_pypy_raw_address_base(self):
+        import sys
+        if '__pypy__' not in sys.modules:
+            skip('PyPy-only test')
         a = memoryview(b"foobar")._pypy_raw_address()
         assert a != 0
         b = memoryview(bytearray(b"foobar"))._pypy_raw_address()
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -12,7 +12,7 @@
     _immutable_ = True
 
     def getlength(self):
-        """Returns the size in bytes (even if getitemsize() > 1)."""
+        """Return the size in bytes."""
         raise NotImplementedError
 
     def __len__(self):

From pypy.commits at gmail.com  Tue May  2 16:07:55 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 02 May 2017 13:07:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Expose msvcrt.SetErrorMode
Message-ID: <5908e71b.223aed0a.dc2f1.94b0@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91173:b2b8897fa2d1
Date: 2017-05-02 22:06 +0200
http://bitbucket.org/pypy/pypy/changeset/b2b8897fa2d1/

Log:	Expose msvcrt.SetErrorMode

diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -122,3 +122,9 @@
 def ungetwch(ch):
     if _lib._ungetwch(ord(ch)) == -1:   # EOF
         _ioerr()
+
+SetErrorMode = _lib.SetErrorMode
+SEM_FAILCRITICALERRORS     = _lib.SEM_FAILCRITICALERRORS
+SEM_NOGPFAULTERRORBOX      = _lib.SEM_NOGPFAULTERRORBOX
+SEM_NOALIGNMENTFAULTEXCEPT = _lib.SEM_NOALIGNMENTFAULTEXCEPT
+SEM_NOOPENFILEERRORBOX     = _lib.SEM_NOOPENFILEERRORBOX

From pypy.commits at gmail.com  Tue May  2 17:45:25 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 02 May 2017 14:45:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: oops, SetErrorMode is in kernel32.dll
Message-ID: <5908fdf5.a82b370a.a844a.065d@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91174:c170bb968e18
Date: 2017-05-02 23:44 +0200
http://bitbucket.org/pypy/pypy/changeset/c170bb968e18/

Log:	oops, SetErrorMode is in kernel32.dll

diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -16,6 +16,7 @@
 import _rawffi
 from _pypy_winbase_cffi import ffi as _ffi
 _lib = _ffi.dlopen(_rawffi.get_libc().name)
+_kernel32 = _ffi.dlopen('kernel32')
 
 import errno
 
@@ -123,7 +124,7 @@
     if _lib._ungetwch(ord(ch)) == -1:   # EOF
         _ioerr()
 
-SetErrorMode = _lib.SetErrorMode
+SetErrorMode = _kernel32.SetErrorMode
 SEM_FAILCRITICALERRORS     = _lib.SEM_FAILCRITICALERRORS
 SEM_NOGPFAULTERRORBOX      = _lib.SEM_NOGPFAULTERRORBOX
 SEM_NOALIGNMENTFAULTEXCEPT = _lib.SEM_NOALIGNMENTFAULTEXCEPT

From pypy.commits at gmail.com  Wed May  3 02:22:30 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 02 May 2017 23:22:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a fake
 getwindowsversion()._platform_version,
 so that platform.win32_ver() can return something.
Message-ID: <59097726.8834370a.e5f14.5010@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91175:09eee048800a
Date: 2017-05-03 08:20 +0200
http://bitbucket.org/pypy/pypy/changeset/09eee048800a/

Log:	Add a fake getwindowsversion()._platform_version, so that
	platform.win32_ver() can return something. (it will always be the
	"compatibility mode" version, I'm not sure what this really means)

diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -234,6 +234,7 @@
     service_pack_minor = structseqfield(11, "Service Pack minor version number")
     suite_mask = structseqfield(12, "Bit mask identifying available product suites")
     product_type = structseqfield(13, "System product type")
+    _platform_version = structseqfield(14, "Diagnostic version number")
 ''')
 
 
@@ -251,6 +252,9 @@
         space.newint(info[6]),
         space.newint(info[7]),
         space.newint(info[8]),
+        # leave _platform_version empty, platform.py will use the main
+        # version numbers above.
+        space.w_None,
     ])
     return space.call_function(w_windows_version_info, raw_version)
 

From pypy.commits at gmail.com  Wed May  3 05:33:25 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 03 May 2017 02:33:25 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-prolonged-backoff:
 Prolong the reduction in transaction length for a number of validations
 after a conflict
Message-ID: <5909a3e5.0b04c80a.74670.de90@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-prolonged-backoff
Changeset: r2053:b8316d20df70
Date: 2017-05-03 11:33 +0200
http://bitbucket.org/pypy/stmgc/changeset/b8316d20df70/

Log:	Prolong the reduction in transaction length for a number of
	validations after a conflict

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -23,41 +23,57 @@
 uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
 // uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-static float stm_default_relative_transaction_length = 0.000001;
-static float *stm_relative_transaction_length_pointer =
-                                    &stm_default_relative_transaction_length;
+static float stm_relative_transaction_length = 0.000001;
+static int stm_increase_transaction_length_backoff = 0;
+
+static void reset_or_decrease_backoff(bool reset) {
+    int actual_backoff = stm_increase_transaction_length_backoff;
+    int expected_backoff;
+    int value = 5;
+    do {
+        expected_backoff = actual_backoff;
+        if (!reset) {
+            value = actual_backoff - 1;
+        }
+        actual_backoff = __atomic_exchange_n(
+            &stm_increase_transaction_length_backoff, value, __ATOMIC_RELAXED);
+    } while (expected_backoff != actual_backoff && (reset || actual_backoff > 0));
+}
 
 static float get_new_transaction_length(bool aborts, float previous) {
     float new = previous;
     if (aborts) {
+        reset_or_decrease_backoff(true); // reset backoff
         if (previous > 0.000001) {
             new = previous / 2;
         } else if (previous > 0) {
             new = 0;
         }
-    } else {
+    } else if (stm_increase_transaction_length_backoff == 0) {
         if (previous - 0.0000001 < 0) {
             new = 0.000001;
         } else if (previous < 1) {
             new = previous * 2;
         }
+    } else { // not abort and backoff != 0
+        reset_or_decrease_backoff(false); // decrease backoff by one
     }
     return new;
 }
 
 static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) {
-    float actual = *stm_relative_transaction_length_pointer;
+    float actual = stm_relative_transaction_length;
     float expected;
     do {
         expected = actual;
         float new = get_new_transaction_length(aborts, actual);
         __atomic_exchange(
-            stm_relative_transaction_length_pointer, &new, &actual, __ATOMIC_RELAXED);
+            &stm_relative_transaction_length, &new, &actual, __ATOMIC_RELAXED);
     } while (fabs(actual - expected) > 0.0000001);
 }
 
 static void stm_update_transaction_length(void) {
-    float relative_additional_length = *stm_relative_transaction_length_pointer;
+    float relative_additional_length = stm_relative_transaction_length;
     stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES +
         (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
     if (timing_enabled()) {

From pypy.commits at gmail.com  Wed May  3 13:10:57 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 03 May 2017 10:10:57 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a new Buffer.typed_read
 method, which does the same as the current strstorage.py: the idea is that
 this will be implemented by all Buffer subclasses for which it makes sense
 (e.g. ByteArrayBuffer can use the same gc_load_indexed approach as
 StringBuffer;
 and raw buffers should be able to use raw_load or similar). Copy the logic
 and the tests from strstorage.py, which will be deleted later
Message-ID: <590a0f21.05cc370a.efe72.b187@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91177:19988cc7db57
Date: 2017-05-03 19:06 +0200
http://bitbucket.org/pypy/pypy/changeset/19988cc7db57/

Log:	add a new Buffer.typed_read method, which does the same as the
	current strstorage.py: the idea is that this will be implemented by
	all Buffer subclasses for which it makes sense (e.g. ByteArrayBuffer
	can use the same gc_load_indexed approach as StringBuffer; and raw
	buffers should be able to use raw_load or similar). Copy the logic
	and the tests from strstorage.py, which will be deleted later

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -1,11 +1,23 @@
 """
 Buffer protocol support.
 """
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
         nonmoving_raw_ptr_for_resizable_list)
 
 
+class CannotRead(Exception):
+    """
+    Exception raised by Buffer.typed_read in case it is not possible to
+    accomplish the request. This might be because it is not supported by the
+    specific type of buffer, or because of alignment issues.
+    """
+
 class Buffer(object):
     """Abstract base class for buffers."""
     _attrs_ = ['readonly']
@@ -80,6 +92,12 @@
     def releasebuffer(self):
         pass
 
+    #@specialize.??
+    def typed_read(self, TP, byte_offset):
+        raise CannotRead
+
+
+
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
@@ -115,6 +133,19 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
+    #@specialize.??
+    def typed_read(self, TP, byte_offset):
+        # WARNING: the 'byte_offset' is, as its name says, measured in bytes;
+        # however, it should be aligned for TP, otherwise on some platforms this
+        # code will crash!
+        lls = llstr(self.value)
+        base_ofs = (llmemory.offsetof(STR, 'chars') +
+                    llmemory.itemoffsetof(STR.chars, 0))
+        scale_factor = llmemory.sizeof(lltype.Char)
+        return llop.gc_load_indexed(TP, lls, byte_offset,
+                                    scale_factor, base_ofs)
+
+
 class SubBuffer(Buffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
     _immutable_ = True
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,3 +1,6 @@
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import r_singlefloat
 from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
@@ -71,3 +74,43 @@
     assert addr[0] == b'h'
     assert addr[4] == b'o'
     assert addr[6] == b'w'
+
+
+class BaseTypedReadTest:
+
+    def test_signed(self):
+        buf = struct.pack('@ll', 42, 43)
+        size = struct.calcsize('@l')
+        assert self.read(lltype.Signed, buf, 0) == 42
+        assert self.read(lltype.Signed, buf, size) == 43
+
+    def test_short(self):
+        buf = struct.pack('@hh', 42, 43)
+        size = struct.calcsize('@h')
+        x = self.read(rffi.SHORT, buf, 0)
+        assert int(x) == 42
+        x = self.read(rffi.SHORT, buf, size)
+        assert int(x) == 43
+
+    def test_float(self):
+        buf = struct.pack('@dd', 12.3, 45.6)
+        size = struct.calcsize('@d')
+        assert self.read(lltype.Float, buf, 0) == 12.3
+        assert self.read(lltype.Float, buf, size) == 45.6
+
+    def test_singlefloat(self):
+        buf = struct.pack('@ff', 12.3, 45.6)
+        size = struct.calcsize('@f')
+        x = self.read(lltype.SingleFloat, buf, 0)
+        assert x == r_singlefloat(12.3)
+        x = self.read(lltype.SingleFloat, buf, size)
+        assert x == r_singlefloat(45.6)
+
+
+class TestTypedReadDirect(BaseTypedReadTest):
+
+    def read(self, TYPE, data, offset):
+        buf = StringBuffer(data)
+        return buf.typed_read(TYPE, offset)
+
+

From pypy.commits at gmail.com  Wed May  3 13:10:59 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 03 May 2017 10:10:59 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add the compiled version of
 the tests. We still need to add the correct @specialize to
 Buffer.typed_read,
 although the tests pass anyway (because we always call Buffer.typed_read
 once)
Message-ID: <590a0f23.832c370a.11ccd.a6ba@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91178:29b4a2158631
Date: 2017-05-03 19:10 +0200
http://bitbucket.org/pypy/pypy/changeset/29b4a2158631/

Log:	add the compiled version of the tests. We still need to add the
	correct @specialize to Buffer.typed_read, although the tests pass
	anyway (because we always call Buffer.typed_read once)

diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -4,7 +4,7 @@
 from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
-
+from rpython.rtyper.test.tool import BaseRtypingTest
 
 def test_string_buffer():
     buf = StringBuffer('hello world')
@@ -114,3 +114,27 @@
         return buf.typed_read(TYPE, offset)
 
 
+class TestCompiled(BaseTypedReadTest):
+    cache = {}
+
+    def read(self, TYPE, data, offset):
+        if TYPE not in self.cache:
+            from rpython.translator.c.test.test_genc import compile
+
+            assert isinstance(TYPE, lltype.Primitive)
+            if TYPE in (lltype.Float, lltype.SingleFloat):
+                TARGET_TYPE = lltype.Float
+            else:
+                TARGET_TYPE = lltype.Signed
+
+            def llf(data, offset):
+                buf = StringBuffer(data)
+                x = buf.typed_read(TYPE, offset)
+                return lltype.cast_primitive(TARGET_TYPE, x)
+
+            fn = compile(llf, [str, int])
+            self.cache[TYPE] = fn
+        #
+        fn = self.cache[TYPE]
+        x = fn(data, offset)
+        return lltype.cast_primitive(TYPE, x)

From pypy.commits at gmail.com  Wed May  3 13:10:54 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 03 May 2017 10:10:54 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: start a branch in which to
 make rstruct faster in a more general way,
 by using llops.gc_load_indexed whenever it's possible,
 not only when reading from strings
Message-ID: <590a0f1e.458e370a.30555.afec@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91176:8b4330b7b7bc
Date: 2017-05-03 12:03 +0200
http://bitbucket.org/pypy/pypy/changeset/8b4330b7b7bc/

Log:	start a branch in which to make rstruct faster in a more general
	way, by using llops.gc_load_indexed whenever it's possible, not only
	when reading from strings


From pypy.commits at gmail.com  Thu May  4 16:28:11 2017
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 04 May 2017 13:28:11 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update qcgc source files
Message-ID: <590b8edb.c773370a.f9b72.b387@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r91179:4cee2f816dc0
Date: 2017-05-04 22:27 +0200
http://bitbucket.org/pypy/pypy/changeset/4cee2f816dc0/

Log:	Update qcgc source files

diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -40,6 +40,10 @@
 	qcgc_hbtable_initialize();
 	qcgc_event_logger_initialize();
 
+#if LOG_ALLOCATOR_SWITCH
+	qcgc_allocations = 0;
+#endif
+
 	env_or_fallback(qcgc_state.incmark_threshold,
 			"QCGC_INCMARK", QCGC_INCMARK_THRESHOLD);
 	env_or_fallback(qcgc_state.incmark_to_sweep,
@@ -49,6 +53,18 @@
 }
 
 void qcgc_destroy(void) {
+#if LOG_ALLOCATOR_SWITCH
+	struct log_info_s {
+		bool bump_allocator;
+		size_t allocations;
+	};
+	struct log_info_s log_info = {
+		_qcgc_bump_allocator.ptr != NULL,
+		qcgc_allocations,
+	};
+	qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
+			(uint8_t *) &log_info);
+#endif
 	qcgc_event_logger_destroy();
 	qcgc_hbtable_destroy();
 	qcgc_allocator_destroy();
@@ -131,12 +147,12 @@
 			if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
 				// Allocator switched
 				struct log_info_s {
+					bool bump_allocator;
 					size_t allocations;
-					bool fit_allocator;
 				};
 				struct log_info_s log_info = {
+					_qcgc_bump_allocator.ptr != NULL,
 					qcgc_allocations,
-					_qcgc_bump_allocator.ptr == NULL,
 				};
 				qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
 						(uint8_t *) &log_info);
@@ -154,12 +170,12 @@
 		if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
 			// Allocator switched
 			struct log_info_s {
+				bool bump_allocator;
 				size_t allocations;
-				bool fit_allocator;
 			};
 			struct log_info_s log_info = {
+				_qcgc_bump_allocator.ptr != NULL,
 				qcgc_allocations,
-				_qcgc_bump_allocator.ptr == NULL,
 			};
 			qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
 					(uint8_t *) &log_info);
@@ -188,12 +204,12 @@
 	if ((_qcgc_bump_allocator.ptr == NULL) != old_use_fit_allocator) {
 		// Allocator switched
 		struct log_info_s {
+			bool bump_allocator;
 			size_t allocations;
-			bool fit_allocator;
 		};
 		struct log_info_s log_info = {
+			_qcgc_bump_allocator.ptr != NULL,
 			qcgc_allocations,
-			_qcgc_bump_allocator.ptr == NULL,
 		};
 		qcgc_event_logger_log(EVENT_ALLOCATOR_SWITCH, sizeof(struct log_info_s),
 				(uint8_t *) &log_info);
diff --git a/rpython/translator/c/src/qcgc/qcgc.h b/rpython/translator/c/src/qcgc/qcgc.h
--- a/rpython/translator/c/src/qcgc/qcgc.h
+++ b/rpython/translator/c/src/qcgc/qcgc.h
@@ -60,7 +60,7 @@
 } object_stack_t;
 
 #if LOG_ALLOCATOR_SWITCH
-size_t qcgc_allocations = 0;
+size_t qcgc_allocations;
 #endif
 
 /**
@@ -289,6 +289,6 @@
  * @param	object	The object to trace
  * @param	visit	The function to be called on the referenced objects
  */
-void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
+extern void qcgc_trace_cb(object_t *object, void (*visit)(object_t *object));
 
 #endif
diff --git a/rpython/translator/c/src/qcgc/src/collector.c b/rpython/translator/c/src/qcgc/src/collector.c
--- a/rpython/translator/c/src/qcgc/src/collector.c
+++ b/rpython/translator/c/src/qcgc/src/collector.c
@@ -155,7 +155,7 @@
 	qcgc_trace_cb(object, &qcgc_push_object);
 }
 
-QCGC_STATIC QCGC_INLINE void qcgc_push_object(object_t *object) {
+QCGC_STATIC void qcgc_push_object(object_t *object) {
 #if CHECKED
 	assert(qcgc_state.phase == GC_MARK);
 #endif

From pypy.commits at gmail.com  Fri May  5 05:40:12 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 05 May 2017 02:40:12 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-prolonged-backoff:
 Backed out changeset 6b4b7aedc3d1
Message-ID: <590c487c.2235c80a.57956.bfb0@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-prolonged-backoff
Changeset: r2055:707bc6a2c0a8
Date: 2017-05-04 12:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/707bc6a2c0a8/

Log:	Backed out changeset 6b4b7aedc3d1

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -44,9 +44,17 @@
     float new = previous;
     if (aborts) {
         reset_or_decrease_backoff(true); // reset backoff
-        new = 0;
+        if (previous > 0.000001) {
+            new = previous / 2;
+        } else if (previous > 0) {
+            new = 0;
+        }
     } else if (stm_increase_transaction_length_backoff == 0) {
-        new = 1;
+        if (previous - 0.0000001 < 0) {
+            new = 0.000001;
+        } else if (previous < 1) {
+            new = previous * 2;
+        }
     } else { // not abort and backoff != 0
         reset_or_decrease_backoff(false); // decrease backoff by one
     }

From pypy.commits at gmail.com  Fri May  5 05:40:14 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 05 May 2017 02:40:14 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-prolonged-backoff: Set
 exponential base of trx length modifier to 1000
Message-ID: <590c487e.6c35ed0a.24705.0bc5@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-prolonged-backoff
Changeset: r2056:22ebd85e36fd
Date: 2017-05-04 17:38 +0200
http://bitbucket.org/pypy/stmgc/changeset/22ebd85e36fd/

Log:	Set exponential base of trx length modifier to 1000

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -17,13 +17,14 @@
 #define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4)
 // just double the size at max
 // #define LARGE_FILL_MARK_NURSERY_BYTES   DEFAULT_FILL_MARK_NURSERY_BYTES
-#define LARGE_FILL_MARK_NURSERY_BYTES   0x10000000000L
+#define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000L
 // #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
 
 uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
 // uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-static float stm_relative_transaction_length = 0.000001;
+#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.000000001f)
+static float stm_relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
 static int stm_increase_transaction_length_backoff = 0;
 
 static void reset_or_decrease_backoff(bool reset) {
@@ -41,19 +42,20 @@
 }
 
 static float get_new_transaction_length(bool aborts, float previous) {
+    const int multiplier = 1000;
     float new = previous;
     if (aborts) {
         reset_or_decrease_backoff(true); // reset backoff
-        if (previous > 0.000001) {
-            new = previous / 2;
-        } else if (previous > 0) {
+        if (previous > STM_MIN_RELATIVE_TRANSACTION_LENGTH) {
+            new = previous / multiplier;
+        } else {
             new = 0;
         }
     } else if (stm_increase_transaction_length_backoff == 0) {
-        if (previous - 0.0000001 < 0) {
-            new = 0.000001;
+        if (previous - (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1) < 0) {
+            new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
         } else if (previous < 1) {
-            new = previous * 2;
+            new = previous * multiplier;
         }
     } else { // not abort and backoff != 0
         reset_or_decrease_backoff(false); // decrease backoff by one
@@ -69,7 +71,7 @@
         float new = get_new_transaction_length(aborts, actual);
         __atomic_exchange(
             &stm_relative_transaction_length, &new, &actual, __ATOMIC_RELAXED);
-    } while (fabs(actual - expected) > 0.0000001);
+    } while (fabs(actual - expected) > (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1));
 }
 
 static void stm_update_transaction_length(void) {

From pypy.commits at gmail.com  Fri May  5 05:40:09 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 05 May 2017 02:40:09 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-prolonged-backoff: Use
 simple on off mechanism for the adaptive mode
Message-ID: <590c4879.879d370a.bb47d.f646@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-prolonged-backoff
Changeset: r2054:6b4b7aedc3d1
Date: 2017-05-04 12:24 +0200
http://bitbucket.org/pypy/stmgc/changeset/6b4b7aedc3d1/

Log:	Use simple on off mechanism for the adaptive mode

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -44,17 +44,9 @@
     float new = previous;
     if (aborts) {
         reset_or_decrease_backoff(true); // reset backoff
-        if (previous > 0.000001) {
-            new = previous / 2;
-        } else if (previous > 0) {
-            new = 0;
-        }
+        new = 0;
     } else if (stm_increase_transaction_length_backoff == 0) {
-        if (previous - 0.0000001 < 0) {
-            new = 0.000001;
-        } else if (previous < 1) {
-            new = previous * 2;
-        }
+        new = 1;
     } else { // not abort and backoff != 0
         reset_or_decrease_backoff(false); // decrease backoff by one
     }

From pypy.commits at gmail.com  Fri May  5 05:40:17 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 05 May 2017 02:40:17 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-prolonged-backoff: Set
 exponential base back to two because the effect was insignificant
Message-ID: <590c4881.d130c80a.87a9.acc8@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-prolonged-backoff
Changeset: r2057:8170562e2157
Date: 2017-05-05 11:39 +0200
http://bitbucket.org/pypy/stmgc/changeset/8170562e2157/

Log:	Set exponential base back to two because the effect was
	insignificant

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -17,13 +17,13 @@
 #define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4)
 // just double the size at max
 // #define LARGE_FILL_MARK_NURSERY_BYTES   DEFAULT_FILL_MARK_NURSERY_BYTES
-#define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000L
+#define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000L
 // #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
 
 uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
 // uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.000000001f)
+#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.000001f)
 static float stm_relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
 static int stm_increase_transaction_length_backoff = 0;
 
@@ -42,7 +42,7 @@
 }
 
 static float get_new_transaction_length(bool aborts, float previous) {
-    const int multiplier = 1000;
+    const int multiplier = 2;
     float new = previous;
     if (aborts) {
         reset_or_decrease_backoff(true); // reset backoff

From pypy.commits at gmail.com  Fri May  5 06:09:40 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 03:09:40 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: specialize on TP
Message-ID: <590c4f64.8ac6370a.2edd.ffc9@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91180:d73762117bd5
Date: 2017-05-04 16:22 +0200
http://bitbucket.org/pypy/pypy/changeset/d73762117bd5/

Log:	specialize on TP

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -92,7 +92,7 @@
     def releasebuffer(self):
         pass
 
-    #@specialize.??
+    @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
         raise CannotRead
 
@@ -133,7 +133,7 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
-    #@specialize.??
+    @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
         # WARNING: the 'byte_offset' is, as its name says, measured in bytes;
         # however, it should be aligned for TP, otherwise on some platforms this

From pypy.commits at gmail.com  Fri May  5 06:09:43 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 03:09:43 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: refactor rstruct.runpack to
 use the new Buffer.typed_read functionality,
 instead of deprecated strstorage. This breaks pypy/module/struct, whstruct,
 which will be fixed later
Message-ID: <590c4f67.50c6370a.fd48.9f75@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91181:474ad89b70a8
Date: 2017-05-04 18:46 +0200
http://bitbucket.org/pypy/pypy/changeset/474ad89b70a8/

Log:	refactor rstruct.runpack to use the new Buffer.typed_read
	functionality, instead of deprecated strstorage. This breaks
	pypy/module/struct, whstruct, which will be fixed later

diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -8,17 +8,20 @@
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.buffer import StringBuffer
 
 class MasterReader(object):
     def __init__(self, s):
-        self.input = s
+        self.inputbuf = StringBuffer(s)
+        self.length = len(s)
         self.inputpos = 0
 
     def read(self, count):
         end = self.inputpos + count
-        if end > len(self.input):
+        if end > self.length:
             raise StructError("unpack str size too short for format")
-        s = self.input[self.inputpos : end]
+        size = end - self.inputpos
+        s = self.inputbuf.getslice(self.inputpos, end, 1, size)
         self.inputpos = end
         return s
 
@@ -40,8 +43,8 @@
         def appendobj(self, value):
             self.value = value
 
-        def get_buffer_as_string_maybe(self):
-            return self.mr.input, self.mr.inputpos
+        def get_buffer_and_pos(self):
+            return self.mr.inputbuf, self.mr.inputpos
 
         def skip(self, size):
             self.read(size) # XXX, could avoid taking the slice
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -14,6 +14,7 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.strstorage import str_storage_getitem
 from rpython.rlib import rarithmetic
+from rpython.rlib.buffer import CannotRead
 from rpython.rtyper.lltypesystem import rffi
 
 native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
@@ -134,19 +135,22 @@
 USE_FASTPATH = True    # set to False by some tests
 ALLOW_SLOWPATH = True  # set to False by some tests
 
-class CannotUnpack(Exception):
-    pass
 
 @specialize.memo()
 def unpack_fastpath(TYPE):
     @specialize.argtype(0)
     def do_unpack_fastpath(fmtiter):
         size = rffi.sizeof(TYPE)
-        strbuf, pos = fmtiter.get_buffer_as_string_maybe()
-        if strbuf is None or pos % size != 0 or not USE_FASTPATH:
-            raise CannotUnpack
+        buf, pos = fmtiter.get_buffer_and_pos()
+        if pos % size != 0 or not USE_FASTPATH:
+            # XXX: maybe we are too conservative here? On most architectures,
+            # it is possible to read the data even if pos is not
+            # aligned. Also, probably it should responsibility of
+            # buf.typed_read to raise CannotRead in case it is not aligned
+            # *and* it is not supported.
+            raise CannotRead
         fmtiter.skip(size)
-        return str_storage_getitem(TYPE, strbuf, pos)
+        return buf.typed_read(TYPE, pos)
     return do_unpack_fastpath
 
 @specialize.argtype(0)
@@ -196,7 +200,7 @@
         try:
             # fast path
             val = unpack_fastpath(TYPE)(fmtiter)
-        except CannotUnpack:
+        except CannotRead:
             # slow path, take the slice
             input = fmtiter.read(size)
             val = str_storage_getitem(TYPE, input, 0)
@@ -251,7 +255,7 @@
             return False
         try:
             intvalue = unpack_fastpath(TYPE)(fmtiter)
-        except CannotUnpack:
+        except CannotRead:
             return False
         if not signed and size < native_int_size:
             intvalue = rarithmetic.intmask(intvalue)

From pypy.commits at gmail.com  Fri May  5 06:09:46 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 03:09:46 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix module/struct to use the
 new functionality. Add a passing test to check that struct.unpack(...,
 <str>) uses the fast-path; test_unpack_from still failing though
Message-ID: <590c4f6a.44ba370a.1c855.e0ba@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91182:1a2232f24cad
Date: 2017-05-04 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/1a2232f24cad/

Log:	fix module/struct to use the new functionality. Add a passing test
	to check that struct.unpack(..., <str>) uses the fast-path;
	test_unpack_from still failing though

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -182,9 +182,8 @@
     def get_pos(self):
         return self.pos
 
-    def get_buffer_as_string_maybe(self):
-        string, pos = self.buf.as_str_and_offset_maybe()
-        return string, pos+self.pos
+    def get_buffer_and_pos(self):
+        return self.buf, self.pos
 
     def skip(self, size):
         self.read(size) # XXX, could avoid taking the slice
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -499,6 +499,10 @@
         from rpython.rlib.rstruct import standardfmttable
         standardfmttable.ALLOW_SLOWPATH = True
 
+    def test_unpack_simple(self):
+        buf = self.struct.pack("iii", 0, 42, 43)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
     def test_unpack_from(self):
         buf = self.struct.pack("iii", 0, 42, 43)
         offset = self.struct.calcsize("i")
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -37,6 +37,7 @@
         # May be overridden.
         return self.getslice(0, self.getlength(), 1, self.getlength())
 
+    # XXX kill me
     def as_str_and_offset_maybe(self):
         """
         If the buffer is backed by a string, return a pair (string, offset), where
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -149,8 +149,11 @@
             # buf.typed_read to raise CannotRead in case it is not aligned
             # *and* it is not supported.
             raise CannotRead
+        # we need to call skip *after* we typed_read(), because if it raises
+        # we do not want to skip
+        result = buf.typed_read(TYPE, pos)
         fmtiter.skip(size)
-        return buf.typed_read(TYPE, pos)
+        return result
     return do_unpack_fastpath
 
 @specialize.argtype(0)

From pypy.commits at gmail.com  Fri May  5 06:09:49 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 03:09:49 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement and test
 SubBuffer.typed_read;
 this fixes pypy/module/struct/test/test_struct.py:test_unpack_from
Message-ID: <590c4f6d.05b4370a.76379.e6f6@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91183:ed8803b41b6c
Date: 2017-05-04 19:24 +0200
http://bitbucket.org/pypy/pypy/changeset/ed8803b41b6c/

Log:	implement and test SubBuffer.typed_read; this fixes
	pypy/module/struct/test/test_struct.py:test_unpack_from

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -207,3 +207,7 @@
         from rpython.rtyper.lltypesystem import rffi
         ptr = self.buffer.get_raw_address()
         return rffi.ptradd(ptr, self.offset)
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        return self.buffer.typed_read(TP, byte_offset + self.offset)
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -114,6 +114,14 @@
         return buf.typed_read(TYPE, offset)
 
 
+class TestSubBufferTypedReadDirect(BaseTypedReadTest):
+
+    def read(self, TYPE, data, offset):
+        buf = StringBuffer('xx' + data)
+        subbuf = SubBuffer(buf, 2, len(data))
+        return subbuf.typed_read(TYPE, offset)
+
+
 class TestCompiled(BaseTypedReadTest):
     cache = {}
 

From pypy.commits at gmail.com  Fri May  5 06:09:52 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 03:09:52 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: kill as_str_and_offset_maybe,
 as it is no longer needed/used anywhere
Message-ID: <590c4f70.9c24c80a.93d1.011e@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91184:a3cde44d8c93
Date: 2017-05-04 19:26 +0200
http://bitbucket.org/pypy/pypy/changeset/a3cde44d8c93/

Log:	kill as_str_and_offset_maybe, as it is no longer needed/used
	anywhere

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -37,15 +37,6 @@
         # May be overridden.
         return self.getslice(0, self.getlength(), 1, self.getlength())
 
-    # XXX kill me
-    def as_str_and_offset_maybe(self):
-        """
-        If the buffer is backed by a string, return a pair (string, offset), where
-        offset is the offset inside the string where the buffer start.
-        Else, return (None, 0).
-        """
-        return None, 0
-
     def getitem(self, index):
         "Returns the index'th character in the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.
@@ -113,9 +104,6 @@
     def as_str(self):
         return self.value
 
-    def as_str_and_offset_maybe(self):
-        return self.value, 0
-
     def getitem(self, index):
         return self.value[index]
 
@@ -178,12 +166,6 @@
         else:
             return 0
 
-    def as_str_and_offset_maybe(self):
-        string, offset = self.buffer.as_str_and_offset_maybe()
-        if string is not None:
-            return string, offset+self.offset
-        return None, 0
-
     def getitem(self, index):
         return self.buffer.getitem(self.offset + index)
 
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -37,31 +37,6 @@
     assert s == SomeInteger(nonneg=True)
 
 
-def test_as_str_and_offset_maybe():
-    buf = StringBuffer('hello world')
-    assert buf.as_str_and_offset_maybe() == ('hello world', 0)
-    #
-    sbuf = SubBuffer(buf, 6, 5)
-    assert sbuf.getslice(0, 5, 1, 5) == 'world'
-    assert sbuf.as_str_and_offset_maybe() == ('hello world', 6)
-    #
-    ssbuf = SubBuffer(sbuf, 3, 2)
-    assert ssbuf.getslice(0, 2, 1, 2) == 'ld'
-    assert ssbuf.as_str_and_offset_maybe() == ('hello world', 9)
-    #
-    ss2buf = SubBuffer(sbuf, 1, -1)
-    assert ss2buf.as_str() == 'orld'
-    assert ss2buf.getlength() == 4
-    ss3buf = SubBuffer(ss2buf, 1, -1)
-    assert ss3buf.as_str() == 'rld'
-    assert ss3buf.getlength() == 3
-    #
-    ss4buf = SubBuffer(buf, 3, 4)
-    assert ss4buf.as_str() == 'lo w'
-    ss5buf = SubBuffer(ss4buf, 1, -1)
-    assert ss5buf.as_str() == 'o w'
-    assert ss5buf.getlength() == 3
-
 def test_repeated_subbuffer():
     buf = StringBuffer('x' * 10000)
     for i in range(9999, 9, -1):

From pypy.commits at gmail.com  Fri May  5 11:29:35 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 08:29:35 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: if we don't ensure that
 StringBuffer.value is not None,
 the annotator crashes as soon as you call str2chap(buf.as_str()). There are
 lots of places which do the assert before calling StringBuffer(),
 but I think it's simpler and better to just do it in the __init__,
 as it has no chances to work if value is None
Message-ID: <590c9a5f.c7dd370a.e9f21.36b7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91185:037cc12c66ea
Date: 2017-05-05 15:29 +0000
http://bitbucket.org/pypy/pypy/changeset/037cc12c66ea/

Log:	if we don't ensure that StringBuffer.value is not None, the
	annotator crashes as soon as you call str2chap(buf.as_str()). There
	are lots of places which do the assert before calling
	StringBuffer(), but I think it's simpler and better to just do it in
	the __init__, as it has no chances to work if value is None

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -95,6 +95,7 @@
     _immutable_ = True
 
     def __init__(self, value):
+        assert value  is not None
         self.value = value
         self.readonly = 1
 

From pypy.commits at gmail.com  Fri May  5 12:46:34 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 09:46:34 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add docstring
Message-ID: <590cac6a.2536ed0a.97f41.2b48@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91186:49b58714e115
Date: 2017-05-05 15:24 +0200
http://bitbucket.org/pypy/pypy/changeset/49b58714e115/

Log:	add docstring

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -86,6 +86,9 @@
 
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
+        """
+        Read the value of type TP starting at byte_offset. No bounds checks
+        """
         raise CannotRead
 
 

From pypy.commits at gmail.com  Fri May  5 12:46:36 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 05 May 2017 09:46:36 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement typed_read for
 BytearrayBuffer and test that struct.unpack takes the fast path. However
 this is sub-optimal, see the comment for an explanation
Message-ID: <590cac6c.4330c80a.bac4a.5968@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91187:3542b2943610
Date: 2017-05-05 18:44 +0200
http://bitbucket.org/pypy/pypy/changeset/3542b2943610/

Log:	implement typed_read for BytearrayBuffer and test that struct.unpack
	takes the fast path. However this is sub-optimal, see the comment
	for an explanation

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -507,3 +507,8 @@
         buf = self.struct.pack("iii", 0, 42, 43)
         offset = self.struct.calcsize("i")
         assert self.struct.unpack_from("ii", buf, offset) == (42, 43)
+
+    def test_unpack_bytearray(self):
+        buf = self.struct.pack("iii", 0, 42, 43)
+        buf = bytearray(buf)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1251,6 +1251,18 @@
     def get_raw_address(self):
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        # XXX: this is sub-efficient, because it forces the bytearray to
+        # become non-movable in order to do the raw_load. In theory, it should
+        # be possible to do the same using llop.gc_load_indexed, the same way
+        # we do it for strings. However, we cannot do it because there is no
+        # way to convert self.data from being a high-level list into the ll
+        # equivalent.
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        raw_ptr = self.get_raw_address()
+        return llop.raw_load(TP, raw_ptr, byte_offset)
+
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):

From pypy.commits at gmail.com  Fri May  5 12:49:31 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Fri, 05 May 2017 09:49:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Populate tp_descr_get and tp_descr_set
 slots
Message-ID: <590cad1b.2239c80a.85b83.4871@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r91188:5b724c72fb87
Date: 2017-05-01 02:07 +0200
http://bitbucket.org/pypy/pypy/changeset/5b724c72fb87/

Log:	Populate tp_descr_get and tp_descr_set slots

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0

From pypy.commits at gmail.com  Fri May  5 12:49:34 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Fri, 05 May 2017 09:49:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue 2551: Struct should be
 initialized in __init__, not __new__
Message-ID: <590cad1e.2235c80a.57956.137c@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r91189:f452cdf49d49
Date: 2017-05-04 00:23 +0200
http://bitbucket.org/pypy/pypy/changeset/f452cdf49d49/

Log:	Issue 2551: Struct should be initialized in __init__, not __new__

diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -115,16 +115,17 @@
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
-    def __init__(self, space, format):
+    format = ""
+    size = -1
+
+    def descr__new__(space, w_subtype, __args__):
+        return space.allocate_instance(W_Struct, w_subtype)
+
+    @unwrap_spec(format='text')
+    def descr__init__(self, space, format):
         self.format = format
         self.size = _calcsize(space, format)
 
-    @unwrap_spec(format='text')
-    def descr__new__(space, w_subtype, format):
-        self = space.allocate_instance(W_Struct, w_subtype)
-        W_Struct.__init__(self, space, format)
-        return self
-
     def descr_pack(self, space, args_w):
         return pack(space, jit.promote_string(self.format), args_w)
 
@@ -141,6 +142,7 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
+    __init__=interp2app(W_Struct.descr__init__),
     format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
     size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -429,6 +429,14 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_struct_subclass(self):
+        class S(self.struct.Struct):
+            def __init__(self):
+                assert self.size == -1
+                super(S, self).__init__('c')
+                assert self.size == 1
+        assert S().unpack('a') == ('a',)
+
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 

From pypy.commits at gmail.com  Fri May  5 13:03:19 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Fri, 05 May 2017 10:03:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: ast.parse(some_ast,
 ...) should return its first argument.
Message-ID: <590cb057.4330c80a.bac4a.5d61@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: 
Changeset: r91190:0a70dce2070a
Date: 2017-05-05 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/0a70dce2070a/

Log:	ast.parse(some_ast, ...) should return its first argument.

diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -50,7 +50,8 @@
         co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """

From pypy.commits at gmail.com  Sat May  6 11:03:44 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 06 May 2017 08:03:44 -0700 (PDT)
Subject: [pypy-commit] pypy default: document that in f(**d) d must contain
 only string keys
Message-ID: <590de5d0.6903c80a.6897e.067d@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91191:7f5084ffd6c2
Date: 2017-05-06 17:03 +0200
http://bitbucket.org/pypy/pypy/changeset/7f5084ffd6c2/

Log:	document that in f(**d) d must contain only string keys

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,11 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even if the called function takes ``**kwargs``. E.g. this code always
+  produces a ``TypeError``, no matter what ``f`` is: ``f(**{1: 2})``.
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.

From pypy.commits at gmail.com  Sat May  6 14:38:47 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 06 May 2017 11:38:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: Bytearrays have amortized constant-time
 "del a[:n]" (backport branch py3.5-bytearray)
Message-ID: <590e1837.4220ed0a.ff286.0f29@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91192:42c69a434735
Date: 2017-05-06 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/42c69a434735/

Log:	Bytearrays have amortized constant-time "del a[:n]" (backport branch
	py3.5-bytearray)

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -4,54 +4,67 @@
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
-from rpython.rlib.debug import check_list_of_chars
+from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
         nonmoving_raw_ptr_for_resizable_list)
+from rpython.rlib import jit
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
-from pypy.interpreter.signature import Signature
 from pypy.interpreter.typedef import TypeDef
-from pypy.objspace.std.sliceobject import W_SliceObject
+from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
+from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
 
 
-
 class W_BytearrayObject(W_Root):
     import_from_mixin(StringMethods)
+    _KIND1 = "bytearray"
+    _KIND2 = "bytearray"
 
     def __init__(self, data):
         check_list_of_chars(data)
-        self.data = resizable_list_supporting_raw_ptr(data)
+        self._data = resizable_list_supporting_raw_ptr(data)
+        self._offset = 0
+        # NOTE: the bytearray data is in 'self._data[self._offset:]'
+        check_nonneg(self._offset)
+        _tweak_for_tests(self)
+
+    def getdata(self):
+        if self._offset > 0:
+            self._data = self._data[self._offset:]
+            self._offset = 0
+        return self._data
 
     def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%s)" % (self.__class__.__name__, ''.join(self.data))
+        return "%s(%s)" % (self.__class__.__name__,
+                           ''.join(self._data[self._offset:]))
 
     def buffer_w(self, space, flags):
-        return BytearrayBuffer(self.data, False)
+        return BytearrayBuffer(self)
 
     def readbuf_w(self, space):
-        return BytearrayBuffer(self.data, True)
+        return BytearrayBuffer(self, readonly=True)
 
     def writebuf_w(self, space):
-        return BytearrayBuffer(self.data, False)
+        return BytearrayBuffer(self)
 
     def charbuf_w(self, space):
-        return ''.join(self.data)
+        return ''.join(self.getdata())
 
     def bytearray_list_of_chars_w(self, space):
-        return self.data
+        return self.getdata()
 
     def nonmovable_carray(self, space):
-        return BytearrayBuffer(self.data, False).get_raw_address()
+        return BytearrayBuffer(self).get_raw_address()
 
     def _new(self, value):
-        if value is self.data:
+        if value is self._data:
             value = value[:]
         return W_BytearrayObject(value)
 
@@ -65,17 +78,27 @@
         return W_BytearrayObject([])
 
     def _len(self):
-        return len(self.data)
+        return len(self._data) - self._offset
+
+    def _fixindex(self, space, index, errmsg="bytearray index out of range"):
+        # for getitem/setitem/delitem of a single char
+        if index >= 0:
+            index += self._offset
+            if index >= len(self._data):
+                raise OperationError(space.w_IndexError, space.newtext(errmsg))
+        else:
+            index += len(self._data)    # count from the end
+            if index < self._offset:
+                raise OperationError(space.w_IndexError, space.newtext(errmsg))
+        check_nonneg(index)
+        return index
 
     def _getitem_result(self, space, index):
-        try:
-            character = self.data[index]
-        except IndexError:
-            raise oefmt(space.w_IndexError, "bytearray index out of range")
+        character = self._data[self._fixindex(space, index)]
         return space.newint(ord(character))
 
     def _val(self, space):
-        return self.data
+        return self.getdata()
 
     @staticmethod
     def _use_rstr_ops(space, w_other):
@@ -152,11 +175,12 @@
         return 1
 
     def ord(self, space):
-        if len(self.data) != 1:
+        length = self._len()
+        if length != 1:
             raise oefmt(space.w_TypeError,
                         "ord() expected a character, but string of length %d "
-                        "found", len(self.data))
-        return space.newint(ord(self.data[0]))
+                        "found", length)
+        return space.newint(ord(self._data[self._offset]))
 
     @staticmethod
     def descr_new(space, w_bytearraytype, __args__):
@@ -169,7 +193,7 @@
             w_dict = space.w_None
         return space.newtuple([
             space.type(self), space.newtuple([
-                space.newunicode(''.join(self.data).decode('latin-1')),
+                space.newunicode(''.join(self.getdata()).decode('latin-1')),
                 space.newtext('latin-1')]),
             w_dict])
 
@@ -202,21 +226,25 @@
         else:
             if count < 0:
                 raise oefmt(space.w_ValueError, "bytearray negative count")
-            self.data = resizable_list_supporting_raw_ptr(['\0'] * count)
+            self._data = resizable_list_supporting_raw_ptr(['\0'] * count)
+            self._offset = 0
             return
 
         data = makebytearraydata_w(space, w_source)
-        self.data = resizable_list_supporting_raw_ptr(data)
+        self._data = resizable_list_supporting_raw_ptr(data)
+        self._offset = 0
+        _tweak_for_tests(self)
 
     def descr_repr(self, space):
-        s = self.data
+        s, start, end, _ = self._convert_idx_params(space, None, None)
 
         # Good default if there are no replacements.
-        buf = StringBuilder(len("bytearray(b'')") + len(s))
+        buf = StringBuilder(len("bytearray(b'')") + (end - start))
 
         buf.append("bytearray(b")
         quote = "'"
-        for c in s:
+        for i in range(start, end):
+            c = s[i]
             if c == '"':
                 quote = "'"
                 break
@@ -224,7 +252,7 @@
                 quote = '"'
         buf.append(quote)
 
-        for i in range(len(s)):
+        for i in range(start, end):
             c = s[i]
 
             if c == '\\' or c == "'":
@@ -250,11 +278,11 @@
         return space.newtext(buf.build())
 
     def descr_str(self, space):
-        return space.newtext(''.join(self.data))
+        return space.newtext(''.join(self.getdata()))
 
     def descr_eq(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return space.newbool(self.data == w_other.data)
+            return space.newbool(self.getdata() == w_other.getdata())
 
         try:
             buffer = _get_buffer(space, w_other)
@@ -274,7 +302,7 @@
 
     def descr_ne(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return space.newbool(self.data != w_other.data)
+            return space.newbool(self.getdata() != w_other.getdata())
 
         try:
             buffer = _get_buffer(space, w_other)
@@ -296,7 +324,7 @@
         value = self._val(space)
 
         if isinstance(w_other, W_BytearrayObject):
-            other = w_other.data
+            other = w_other.getdata()
             other_len = len(other)
             cmp = _memcmp(value, other, min(len(value), len(other)))
         elif isinstance(w_other, W_BytesObject):
@@ -344,7 +372,7 @@
 
     def descr_inplace_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            self.data += w_other.data
+            self._data += w_other.getdata()
             return self
 
         if isinstance(w_other, W_BytesObject):
@@ -356,7 +384,7 @@
     @specialize.argtype(1)
     def _inplace_add(self, other):
         for i in range(len(other)):
-            self.data.append(other[i])
+            self._data.append(other[i])
 
     def descr_inplace_mul(self, space, w_times):
         try:
@@ -365,85 +393,99 @@
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
             raise
-        self.data *= times
+        data = self.getdata()
+        data *= times
         return self
 
     def descr_setitem(self, space, w_index, w_other):
         if isinstance(w_index, W_SliceObject):
-            oldsize = len(self.data)
+            sequence2 = makebytearraydata_w(space, w_other)
+            oldsize = self._len()
             start, stop, step, slicelength = w_index.indices4(space, oldsize)
-            sequence2 = makebytearraydata_w(space, w_other)
-            _setitem_slice_helper(space, self.data, start, step,
+            if start == 0 and step == 1 and len(sequence2) <= slicelength:
+                self._delete_from_start(slicelength - len(sequence2))
+                slicelength = len(sequence2)
+                if slicelength == 0:
+                    return
+            data = self._data
+            start += self._offset
+            _setitem_slice_helper(space, data, start, step,
                                   slicelength, sequence2, empty_elem='\x00')
         else:
             idx = space.getindex_w(w_index, space.w_IndexError,
                                    "bytearray index")
-            try:
-                self.data[idx] = getbytevalue(space, w_other)
-            except IndexError:
-                raise oefmt(space.w_IndexError, "bytearray index out of range")
+            newvalue = getbytevalue(space, w_other)
+            self._data[self._fixindex(space, idx)] = newvalue
 
     def descr_delitem(self, space, w_idx):
         if isinstance(w_idx, W_SliceObject):
-            start, stop, step, slicelength = w_idx.indices4(space,
-                                                            len(self.data))
-            _delitem_slice_helper(space, self.data, start, step, slicelength)
+            start, stop, step, slicelength = w_idx.indices4(space, self._len())
+            if start == 0 and step == 1:
+                self._delete_from_start(slicelength)
+            else:
+                _delitem_slice_helper(space, self._data,
+                                      start + self._offset, step, slicelength)
         else:
             idx = space.getindex_w(w_idx, space.w_IndexError,
                                    "bytearray index")
-            try:
-                del self.data[idx]
-            except IndexError:
-                raise oefmt(space.w_IndexError,
-                            "bytearray deletion index out of range")
+            idx = self._fixindex(space, idx)
+            if idx == self._offset:    # fast path for del x[0] or del[-len]
+                self._delete_from_start(1)
+            else:
+                del self._data[idx]
+
+    def _delete_from_start(self, n):
+        assert n >= 0
+        self._offset += n
+        jit.conditional_call(self._offset > len(self._data) / 2,
+                             _shrink_after_delete_from_start, self)
 
     def descr_append(self, space, w_item):
-        self.data.append(getbytevalue(space, w_item))
+        self._data.append(getbytevalue(space, w_item))
 
     def descr_extend(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            self.data += w_other.data
+            self._data += w_other.getdata()
         else:
-            self.data += makebytearraydata_w(space, w_other)
-        return self
+            self._inplace_add(makebytearraydata_w(space, w_other))
 
     def descr_insert(self, space, w_idx, w_other):
         where = space.int_w(w_idx)
-        length = len(self.data)
+        val = getbytevalue(space, w_other)
+        data = self.getdata()
+        length = len(data)
         index = get_positive_index(where, length)
-        val = getbytevalue(space, w_other)
-        self.data.insert(index, val)
-        return space.w_None
+        data.insert(index, val)
 
     @unwrap_spec(w_idx=WrappedDefault(-1))
     def descr_pop(self, space, w_idx):
         index = space.int_w(w_idx)
-        try:
-            result = self.data.pop(index)
-        except IndexError:
-            if not self.data:
-                raise oefmt(space.w_IndexError, "pop from empty bytearray")
-            raise oefmt(space.w_IndexError, "pop index out of range")
+        if self._len() == 0:
+            raise oefmt(space.w_IndexError, "pop from empty bytearray")
+        index = self._fixindex(space, index, "pop index out of range")
+        result = self._data.pop(index)
         return space.newint(ord(result))
 
     def descr_remove(self, space, w_char):
         char = space.int_w(space.index(w_char))
-        try:
-            self.data.remove(chr(char))
-        except ValueError:
-            raise oefmt(space.w_ValueError, "value not found in bytearray")
+        _data = self._data
+        for index in range(self._offset, len(_data)):
+            if ord(_data[index]) == char:
+                del _data[index]
+                return
+        raise oefmt(space.w_ValueError, "value not found in bytearray")
 
     _StringMethods_descr_contains = descr_contains
     def descr_contains(self, space, w_sub):
         if space.isinstance_w(w_sub, space.w_int):
             char = space.int_w(w_sub)
-            return _descr_contains_bytearray(self.data, space, char)
+            return _descr_contains_bytearray(self.getdata(), space, char)
 
         return self._StringMethods_descr_contains(space, w_sub)
 
     def descr_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return self._new(self.data + w_other.data)
+            return self._new(self.getdata() + w_other.getdata())
 
         if isinstance(w_other, W_BytesObject):
             return self._add(self._op_val(space, w_other))
@@ -458,11 +500,37 @@
 
     @specialize.argtype(1)
     def _add(self, other):
-        return self._new(self.data + [other[i] for i in range(len(other))])
+        return self._new(self.getdata() + [other[i] for i in range(len(other))])
 
     def descr_reverse(self, space):
-        self.data.reverse()
+        self.getdata().reverse()
 
+    def descr_alloc(self, space):
+        return space.newint(len(self._data) + 1)   # includes the _offset part
+
+    def _convert_idx_params(self, space, w_start, w_end):
+        # optimization: this version doesn't force getdata()
+        start, end = unwrap_start_stop(space, self._len(), w_start, w_end)
+        ofs = self._offset
+        return (self._data, start + ofs, end + ofs, ofs)
+
+    def descr_getitem(self, space, w_index):
+        # optimization: this version doesn't force getdata()
+        if isinstance(w_index, W_SliceObject):
+            start, stop, step, sl = w_index.indices4(space, self._len())
+            if sl == 0:
+                return self._empty()
+            elif step == 1:
+                assert start >= 0 and stop >= 0
+                ofs = self._offset
+                return self._new(self._data[start + ofs : stop + ofs])
+            else:
+                start += self._offset
+                ret = _descr_getslice_slowpath(self._data, start, step, sl)
+                return self._new_from_list(ret)
+
+        index = space.getindex_w(w_index, space.w_IndexError, self._KIND1)
+        return self._getitem_result(space, index)
 
 
 # ____________________________________________________________
@@ -1133,6 +1201,8 @@
                         doc=BytearrayDocstrings.remove.__doc__),
     reverse = interp2app(W_BytearrayObject.descr_reverse,
                          doc=BytearrayDocstrings.reverse.__doc__),
+    __alloc__ = interp2app(W_BytearrayObject.descr_alloc,
+                           doc=BytearrayDocstrings.__alloc__.__doc__),
 )
 W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
@@ -1197,21 +1267,6 @@
                     "attempt to assign sequence of size %d to extended slice "
                     "of size %d", len2, slicelength)
 
-    if sequence2 is items:
-        if step > 0:
-            # Always copy starting from the right to avoid
-            # having to make a shallow copy in the case where
-            # the source and destination lists are the same list.
-            i = len2 - 1
-            start += i*step
-            while i >= 0:
-                items[start] = sequence2[i]
-                start -= step
-                i -= 1
-            return
-        else:
-            # Make a shallow copy to more easily handle the reversal case
-            sequence2 = list(sequence2)
     for i in range(len2):
         items[start] = sequence2[i]
         start += step
@@ -1220,36 +1275,47 @@
 class BytearrayBuffer(Buffer):
     _immutable_ = True
 
-    def __init__(self, data, readonly):
-        self.data = data
+    def __init__(self, ba, readonly=False):
+        self.ba = ba     # the W_BytearrayObject
         self.readonly = readonly
 
     def getlength(self):
-        return len(self.data)
+        return self.ba._len()
 
     def getitem(self, index):
-        return self.data[index]
+        ba = self.ba
+        return ba._data[ba._offset + index]
 
     def setitem(self, index, char):
-        self.data[index] = char
+        ba = self.ba
+        ba._data[ba._offset + index] = char
 
     def getslice(self, start, stop, step, size):
         if size == 0:
             return ""
         if step == 1:
             assert 0 <= start <= stop
-            if start == 0 and stop == len(self.data):
-                return "".join(self.data)
-            return "".join(self.data[start:stop])
+            ba = self.ba
+            start += ba._offset
+            stop += ba._offset
+            data = ba._data
+            if start != 0 or stop != len(data):
+                data = data[start:stop]
+            return "".join(data)
         return Buffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
+        ba = self.ba
+        start += ba._offset
         for i in range(len(string)):
-            self.data[start + i] = string[i]
+            ba._data[start + i] = string[i]
 
     def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
+        ba = self.ba
+        p = nonmoving_raw_ptr_for_resizable_list(ba._data)
+        p = rffi.ptradd(p, ba._offset)
+        return p
 
 
 @specialize.argtype(1)
@@ -1261,3 +1327,9 @@
         if selfvalue[i] > buffer[i]:
             return 1
     return 0
+
+def _tweak_for_tests(w_bytearray):
+    "Patched in test_bytearray.py"
+
+def _shrink_after_delete_from_start(w_bytearray):
+    w_bytearray.getdata()
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -26,7 +26,8 @@
         value = self._val(space)
         lenself = len(value)
         start, end = unwrap_start_stop(space, lenself, w_start, w_end)
-        return (value, start, end)
+        # the None means "no offset"; see bytearrayobject.py
+        return (value, start, end, None)
 
     def _multi_chr(self, c):
         return c
@@ -38,18 +39,18 @@
     #    pass
 
     def descr_contains(self, space, w_sub):
-        value = self._val(space)
+        value, start, end, _ = self._convert_idx_params(space, None, None)
         if self._use_rstr_ops(space, w_sub):
             other = self._op_val(space, w_sub)
-            return space.newbool(value.find(other) >= 0)
+            return space.newbool(value.find(other, start, end) >= 0)
 
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytesObject):
             other = self._op_val(space, w_sub)
-            res = find(value, other, 0, len(value))
+            res = find(value, other, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
-            res = find(value, buffer, 0, len(value))
+            res = find(value, buffer, start, end)
 
         return space.newbool(res >= 0)
 
@@ -146,7 +147,7 @@
         return self._new(centered)
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
-        value, start, end = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             return space.newint(value.count(self._op_val(space, w_sub), start,
@@ -155,7 +156,7 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = count(value, w_sub.data, start, end)
+            res = count(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = count(value, w_sub._value, start, end)
         else:
@@ -235,7 +236,7 @@
         return distance
 
     def descr_find(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             res = value.find(self._op_val(space, w_sub), start, end)
@@ -244,17 +245,18 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = find(value, w_sub.data, start, end)
+            res = find(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = find(value, w_sub._value, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
             res = find(value, buffer, start, end)
-
+        if ofs is not None and res >= 0:
+            res -= ofs
         return space.newint(res)
 
     def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             res = value.rfind(self._op_val(space, w_sub), start, end)
@@ -263,24 +265,25 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = rfind(value, w_sub.data, start, end)
+            res = rfind(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = rfind(value, w_sub._value, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
             res = rfind(value, buffer, start, end)
-
+        if ofs is not None and res >= 0:
+            res -= ofs
         return space.newint(res)
 
     def descr_index(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if self._use_rstr_ops(space, w_sub):
             res = value.find(self._op_val(space, w_sub), start, end)
         elif isinstance(w_sub, W_BytearrayObject):
-            res = find(value, w_sub.data, start, end)
+            res = find(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = find(value, w_sub._value, start, end)
         else:
@@ -290,17 +293,19 @@
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.index")
+        if ofs is not None:
+            res -= ofs
         return space.newint(res)
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if self._use_rstr_ops(space, w_sub):
             res = value.rfind(self._op_val(space, w_sub), start, end)
         elif isinstance(w_sub, W_BytearrayObject):
-            res = rfind(value, w_sub.data, start, end)
+            res = rfind(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = rfind(value, w_sub._value, start, end)
         else:
@@ -310,6 +315,8 @@
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.rindex")
+        if ofs is not None:
+            res -= ofs
         return space.newint(res)
 
     @specialize.arg(2)
@@ -612,7 +619,7 @@
         return self._newlist_unwrapped(space, strs)
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_prefix, space.w_tuple):
             return self._startswith_tuple(space, value, w_prefix, start, end)
         return space.newbool(self._startswith(space, value, w_prefix, start,
@@ -635,7 +642,7 @@
                          # bytearrays, but overridden for unicodes
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_suffix, space.w_tuple):
             return self._endswith_tuple(space, value, w_suffix, start, end)
         return space.newbool(self._endswith(space, value, w_suffix, start,
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,9 +1,26 @@
+import random
 from pypy import conftest
+from pypy.objspace.std import bytearrayobject
+
+class DontAccess(object):
+    pass
+dont_access = DontAccess()
+
 
 
 class AppTestBytesArray:
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
+        def tweak(w_bytearray):
+            n = random.randint(-3, 16)
+            if n > 0:
+                w_bytearray._data = [dont_access] * n + w_bytearray._data
+                w_bytearray._offset += n
+        cls._old_tweak = [bytearrayobject._tweak_for_tests]
+        bytearrayobject._tweak_for_tests = tweak
+
+    def teardown_class(cls):
+        [bytearrayobject._tweak_for_tests] = cls._old_tweak
 
     def test_basics(self):
         b = bytearray()
@@ -67,6 +84,7 @@
         raises(IndexError, b.__getitem__, 4)
         assert b[1:5] == bytearray('est')
         assert b[slice(1,5)] == bytearray('est')
+        assert b[1:5:2] == bytearray(b'et')
 
     def test_arithmetic(self):
         b1 = bytearray('hello ')
@@ -204,6 +222,10 @@
         assert bytearray('ab').endswith(bytearray(''), 2) is True
         assert bytearray('ab').endswith(bytearray(''), 3) is False
 
+    def test_startswith_self(self):
+        b = bytearray(b'abcd')
+        assert b.startswith(b)
+
     def test_stringlike_conversions(self):
         # methods that should return bytearray (and not str)
         def check(result, expected):
@@ -330,6 +352,20 @@
         b.reverse()
         assert b == bytearray('olleh')
 
+    def test_delitem_from_front(self):
+        b = bytearray(b'abcdefghij')
+        del b[0]
+        del b[0]
+        assert len(b) == 8
+        assert b == bytearray(b'cdefghij')
+        del b[-8]
+        del b[-7]
+        assert len(b) == 6
+        assert b == bytearray(b'efghij')
+        del b[:3]
+        assert len(b) == 3
+        assert b == bytearray(b'hij')
+
     def test_delitem(self):
         b = bytearray('abc')
         del b[1]
@@ -412,6 +448,18 @@
         raises(TypeError, b.extend, [object()])
         raises(TypeError, b.extend, u"unicode")
 
+    def test_setitem_from_front(self):
+        b = bytearray(b'abcdefghij')
+        b[:2] = b''
+        assert len(b) == 8
+        assert b == bytearray(b'cdefghij')
+        b[:3] = b'X'
+        assert len(b) == 6
+        assert b == bytearray(b'Xfghij')
+        b[:2] = b'ABC'
+        assert len(b) == 7
+        assert b == bytearray(b'ABCghij')
+
     def test_setslice(self):
         b = bytearray('hello')
         b[:] = [ord(c) for c in 'world']
@@ -502,3 +550,78 @@
     def test_split_whitespace(self):
         b = bytearray(b'\x09\x0A\x0B\x0C\x0D\x1C\x1D\x1E\x1F')
         assert b.split() == [b'\x1c\x1d\x1e\x1f']
+
+    def test_dont_force_offset(self):
+        def make(x=b'abcdefghij', shift=3):
+            b = bytearray(b'?'*shift + x)
+            b + b''                       # force 'b'
+            del b[:shift]                 # add shift to b._offset
+            return b
+        assert make(shift=0).__alloc__() == 11
+        #
+        x = make(shift=3)
+        assert x.__alloc__() == 14
+        assert memoryview(x)[1] == 'b'
+        assert x.__alloc__() == 14
+        assert len(x) == 10
+        assert x.__alloc__() == 14
+        assert x[3] == ord('d')
+        assert x[-3] == ord('h')
+        assert x.__alloc__() == 14
+        assert x[3:-3] == b'defg'
+        assert x[-3:3:-1] == b'hgfe'
+        assert x.__alloc__() == 14
+        assert repr(x) == "bytearray(b'abcdefghij')"
+        assert x.__alloc__() == 14
+        #
+        x = make(shift=3)
+        x[3] = ord('D')
+        assert x.__alloc__() == 14
+        x[4:6] = b'EF'
+        assert x.__alloc__() == 14
+        x[6:8] = b'G'
+        assert x.__alloc__() == 13
+        x[-2:4:-2] = b'*/'
+        assert x.__alloc__() == 13
+        assert x == bytearray(b'abcDE/G*j')
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        assert len(x) == 26
+        assert x.__alloc__() == 38
+        del x[:1]
+        assert len(x) == 25
+        assert x.__alloc__() == 38
+        del x[0:5]
+        assert len(x) == 20
+        assert x.__alloc__() == 38
+        del x[0]
+        assert len(x) == 19
+        assert x.__alloc__() == 38
+        del x[0]                      # too much emptiness, forces now
+        assert len(x) == 18
+        assert x.__alloc__() == 19
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        del x[:9]                     # too much emptiness, forces now
+        assert len(x) == 17
+        assert x.__alloc__() == 18
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        assert x.__alloc__() == 38
+        del x[1]
+        assert x.__alloc__() == 37      # not forced, but the list shrank
+        del x[3:10:2]
+        assert x.__alloc__() == 33
+        assert x == bytearray(b'acdfhjlmnopqrstuvwxyz')
+        #
+        x = make(shift=3)
+        assert b'f' in x
+        assert b'ef' in x
+        assert b'efx' not in x
+        assert b'very long string longer than the original' not in x
+        assert x.__alloc__() == 14
+        assert x.find(b'f') == 5
+        assert x.rfind(b'f', 2, 11) == 5
+        assert x.find(b'fe') == -1
+        assert x.index(b'f', 2, 11) == 5
+        assert x.__alloc__() == 14

From pypy.commits at gmail.com  Sat May  6 14:38:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 06 May 2017 11:38:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <590e183a.07dee90a.f4c5.2205@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91193:df2a94c7ae6f
Date: 2017-05-06 19:38 +0100
http://bitbucket.org/pypy/pypy/changeset/df2a94c7ae6f/

Log:	merge heads

diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,11 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even if the called function takes ``**kwargs``. E.g. this code always
+  produces a ``TypeError``, no matter what ``f`` is: ``f(**{1: 2})``.
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -50,7 +50,8 @@
         co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -115,16 +115,17 @@
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
-    def __init__(self, space, format):
+    format = ""
+    size = -1
+
+    def descr__new__(space, w_subtype, __args__):
+        return space.allocate_instance(W_Struct, w_subtype)
+
+    @unwrap_spec(format='text')
+    def descr__init__(self, space, format):
         self.format = format
         self.size = _calcsize(space, format)
 
-    @unwrap_spec(format='text')
-    def descr__new__(space, w_subtype, format):
-        self = space.allocate_instance(W_Struct, w_subtype)
-        W_Struct.__init__(self, space, format)
-        return self
-
     def descr_pack(self, space, args_w):
         return pack(space, jit.promote_string(self.format), args_w)
 
@@ -141,6 +142,7 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
+    __init__=interp2app(W_Struct.descr__init__),
     format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
     size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -429,6 +429,14 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_struct_subclass(self):
+        class S(self.struct.Struct):
+            def __init__(self):
+                assert self.size == -1
+                super(S, self).__init__('c')
+                assert self.size == 1
+        assert S().unpack('a') == ('a',)
+
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 

From pypy.commits at gmail.com  Sat May  6 14:51:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 06 May 2017 11:51:51 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: hg merge default
Message-ID: <590e1b47.cf4f370a.ffc09.27fe@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91194:9d07ac7eaf1a
Date: 2017-05-06 19:40 +0100
http://bitbucket.org/pypy/pypy/changeset/9d07ac7eaf1a/

Log:	hg merge default

diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,11 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even if the called function takes ``**kwargs``. E.g. this code always
+  produces a ``TypeError``, no matter what ``f`` is: ``f(**{1: 2})``.
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -50,7 +50,8 @@
         co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -115,16 +115,17 @@
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
-    def __init__(self, space, format):
+    format = ""
+    size = -1
+
+    def descr__new__(space, w_subtype, __args__):
+        return space.allocate_instance(W_Struct, w_subtype)
+
+    @unwrap_spec(format='text')
+    def descr__init__(self, space, format):
         self.format = format
         self.size = _calcsize(space, format)
 
-    @unwrap_spec(format='text')
-    def descr__new__(space, w_subtype, format):
-        self = space.allocate_instance(W_Struct, w_subtype)
-        W_Struct.__init__(self, space, format)
-        return self
-
     def descr_pack(self, space, args_w):
         return pack(space, jit.promote_string(self.format), args_w)
 
@@ -141,6 +142,7 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
+    __init__=interp2app(W_Struct.descr__init__),
     format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
     size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -429,6 +429,14 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_struct_subclass(self):
+        class S(self.struct.Struct):
+            def __init__(self):
+                assert self.size == -1
+                super(S, self).__init__('c')
+                assert self.size == 1
+        assert S().unpack('a') == ('a',)
+
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -4,54 +4,67 @@
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
-from rpython.rlib.debug import check_list_of_chars
+from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
         nonmoving_raw_ptr_for_resizable_list)
+from rpython.rlib import jit
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
-from pypy.interpreter.signature import Signature
 from pypy.interpreter.typedef import TypeDef
-from pypy.objspace.std.sliceobject import W_SliceObject
+from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
+from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.util import get_positive_index
 
 
-
 class W_BytearrayObject(W_Root):
     import_from_mixin(StringMethods)
+    _KIND1 = "bytearray"
+    _KIND2 = "bytearray"
 
     def __init__(self, data):
         check_list_of_chars(data)
-        self.data = resizable_list_supporting_raw_ptr(data)
+        self._data = resizable_list_supporting_raw_ptr(data)
+        self._offset = 0
+        # NOTE: the bytearray data is in 'self._data[self._offset:]'
+        check_nonneg(self._offset)
+        _tweak_for_tests(self)
+
+    def getdata(self):
+        if self._offset > 0:
+            self._data = self._data[self._offset:]
+            self._offset = 0
+        return self._data
 
     def __repr__(self):
         """representation for debugging purposes"""
-        return "%s(%s)" % (self.__class__.__name__, ''.join(self.data))
+        return "%s(%s)" % (self.__class__.__name__,
+                           ''.join(self._data[self._offset:]))
 
     def buffer_w(self, space, flags):
-        return BytearrayBuffer(self.data, False)
+        return BytearrayBuffer(self)
 
     def readbuf_w(self, space):
-        return BytearrayBuffer(self.data, True)
+        return BytearrayBuffer(self, readonly=True)
 
     def writebuf_w(self, space):
-        return BytearrayBuffer(self.data, False)
+        return BytearrayBuffer(self)
 
     def charbuf_w(self, space):
-        return ''.join(self.data)
+        return ''.join(self.getdata())
 
     def bytearray_list_of_chars_w(self, space):
-        return self.data
+        return self.getdata()
 
     def nonmovable_carray(self, space):
-        return BytearrayBuffer(self.data, False).get_raw_address()
+        return BytearrayBuffer(self).get_raw_address()
 
     def _new(self, value):
-        if value is self.data:
+        if value is self._data:
             value = value[:]
         return W_BytearrayObject(value)
 
@@ -65,17 +78,27 @@
         return W_BytearrayObject([])
 
     def _len(self):
-        return len(self.data)
+        return len(self._data) - self._offset
+
+    def _fixindex(self, space, index, errmsg="bytearray index out of range"):
+        # for getitem/setitem/delitem of a single char
+        if index >= 0:
+            index += self._offset
+            if index >= len(self._data):
+                raise OperationError(space.w_IndexError, space.newtext(errmsg))
+        else:
+            index += len(self._data)    # count from the end
+            if index < self._offset:
+                raise OperationError(space.w_IndexError, space.newtext(errmsg))
+        check_nonneg(index)
+        return index
 
     def _getitem_result(self, space, index):
-        try:
-            character = self.data[index]
-        except IndexError:
-            raise oefmt(space.w_IndexError, "bytearray index out of range")
+        character = self._data[self._fixindex(space, index)]
         return space.newint(ord(character))
 
     def _val(self, space):
-        return self.data
+        return self.getdata()
 
     @staticmethod
     def _use_rstr_ops(space, w_other):
@@ -152,11 +175,12 @@
         return 1
 
     def ord(self, space):
-        if len(self.data) != 1:
+        length = self._len()
+        if length != 1:
             raise oefmt(space.w_TypeError,
                         "ord() expected a character, but string of length %d "
-                        "found", len(self.data))
-        return space.newint(ord(self.data[0]))
+                        "found", length)
+        return space.newint(ord(self._data[self._offset]))
 
     @staticmethod
     def descr_new(space, w_bytearraytype, __args__):
@@ -169,7 +193,7 @@
             w_dict = space.w_None
         return space.newtuple([
             space.type(self), space.newtuple([
-                space.newunicode(''.join(self.data).decode('latin-1')),
+                space.newunicode(''.join(self.getdata()).decode('latin-1')),
                 space.newtext('latin-1')]),
             w_dict])
 
@@ -202,21 +226,25 @@
         else:
             if count < 0:
                 raise oefmt(space.w_ValueError, "bytearray negative count")
-            self.data = resizable_list_supporting_raw_ptr(['\0'] * count)
+            self._data = resizable_list_supporting_raw_ptr(['\0'] * count)
+            self._offset = 0
             return
 
         data = makebytearraydata_w(space, w_source)
-        self.data = resizable_list_supporting_raw_ptr(data)
+        self._data = resizable_list_supporting_raw_ptr(data)
+        self._offset = 0
+        _tweak_for_tests(self)
 
     def descr_repr(self, space):
-        s = self.data
+        s, start, end, _ = self._convert_idx_params(space, None, None)
 
         # Good default if there are no replacements.
-        buf = StringBuilder(len("bytearray(b'')") + len(s))
+        buf = StringBuilder(len("bytearray(b'')") + (end - start))
 
         buf.append("bytearray(b")
         quote = "'"
-        for c in s:
+        for i in range(start, end):
+            c = s[i]
             if c == '"':
                 quote = "'"
                 break
@@ -224,7 +252,7 @@
                 quote = '"'
         buf.append(quote)
 
-        for i in range(len(s)):
+        for i in range(start, end):
             c = s[i]
 
             if c == '\\' or c == "'":
@@ -250,11 +278,11 @@
         return space.newtext(buf.build())
 
     def descr_str(self, space):
-        return space.newtext(''.join(self.data))
+        return space.newtext(''.join(self.getdata()))
 
     def descr_eq(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return space.newbool(self.data == w_other.data)
+            return space.newbool(self.getdata() == w_other.getdata())
 
         try:
             buffer = _get_buffer(space, w_other)
@@ -274,7 +302,7 @@
 
     def descr_ne(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return space.newbool(self.data != w_other.data)
+            return space.newbool(self.getdata() != w_other.getdata())
 
         try:
             buffer = _get_buffer(space, w_other)
@@ -296,7 +324,7 @@
         value = self._val(space)
 
         if isinstance(w_other, W_BytearrayObject):
-            other = w_other.data
+            other = w_other.getdata()
             other_len = len(other)
             cmp = _memcmp(value, other, min(len(value), len(other)))
         elif isinstance(w_other, W_BytesObject):
@@ -344,7 +372,7 @@
 
     def descr_inplace_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            self.data += w_other.data
+            self._data += w_other.getdata()
             return self
 
         if isinstance(w_other, W_BytesObject):
@@ -356,7 +384,7 @@
     @specialize.argtype(1)
     def _inplace_add(self, other):
         for i in range(len(other)):
-            self.data.append(other[i])
+            self._data.append(other[i])
 
     def descr_inplace_mul(self, space, w_times):
         try:
@@ -365,85 +393,99 @@
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
             raise
-        self.data *= times
+        data = self.getdata()
+        data *= times
         return self
 
     def descr_setitem(self, space, w_index, w_other):
         if isinstance(w_index, W_SliceObject):
-            oldsize = len(self.data)
+            sequence2 = makebytearraydata_w(space, w_other)
+            oldsize = self._len()
             start, stop, step, slicelength = w_index.indices4(space, oldsize)
-            sequence2 = makebytearraydata_w(space, w_other)
-            _setitem_slice_helper(space, self.data, start, step,
+            if start == 0 and step == 1 and len(sequence2) <= slicelength:
+                self._delete_from_start(slicelength - len(sequence2))
+                slicelength = len(sequence2)
+                if slicelength == 0:
+                    return
+            data = self._data
+            start += self._offset
+            _setitem_slice_helper(space, data, start, step,
                                   slicelength, sequence2, empty_elem='\x00')
         else:
             idx = space.getindex_w(w_index, space.w_IndexError,
                                    "bytearray index")
-            try:
-                self.data[idx] = getbytevalue(space, w_other)
-            except IndexError:
-                raise oefmt(space.w_IndexError, "bytearray index out of range")
+            newvalue = getbytevalue(space, w_other)
+            self._data[self._fixindex(space, idx)] = newvalue
 
     def descr_delitem(self, space, w_idx):
         if isinstance(w_idx, W_SliceObject):
-            start, stop, step, slicelength = w_idx.indices4(space,
-                                                            len(self.data))
-            _delitem_slice_helper(space, self.data, start, step, slicelength)
+            start, stop, step, slicelength = w_idx.indices4(space, self._len())
+            if start == 0 and step == 1:
+                self._delete_from_start(slicelength)
+            else:
+                _delitem_slice_helper(space, self._data,
+                                      start + self._offset, step, slicelength)
         else:
             idx = space.getindex_w(w_idx, space.w_IndexError,
                                    "bytearray index")
-            try:
-                del self.data[idx]
-            except IndexError:
-                raise oefmt(space.w_IndexError,
-                            "bytearray deletion index out of range")
+            idx = self._fixindex(space, idx)
+            if idx == self._offset:    # fast path for del x[0] or del[-len]
+                self._delete_from_start(1)
+            else:
+                del self._data[idx]
+
+    def _delete_from_start(self, n):
+        assert n >= 0
+        self._offset += n
+        jit.conditional_call(self._offset > len(self._data) / 2,
+                             _shrink_after_delete_from_start, self)
 
     def descr_append(self, space, w_item):
-        self.data.append(getbytevalue(space, w_item))
+        self._data.append(getbytevalue(space, w_item))
 
     def descr_extend(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            self.data += w_other.data
+            self._data += w_other.getdata()
         else:
-            self.data += makebytearraydata_w(space, w_other)
-        return self
+            self._inplace_add(makebytearraydata_w(space, w_other))
 
     def descr_insert(self, space, w_idx, w_other):
         where = space.int_w(w_idx)
-        length = len(self.data)
+        val = getbytevalue(space, w_other)
+        data = self.getdata()
+        length = len(data)
         index = get_positive_index(where, length)
-        val = getbytevalue(space, w_other)
-        self.data.insert(index, val)
-        return space.w_None
+        data.insert(index, val)
 
     @unwrap_spec(w_idx=WrappedDefault(-1))
     def descr_pop(self, space, w_idx):
         index = space.int_w(w_idx)
-        try:
-            result = self.data.pop(index)
-        except IndexError:
-            if not self.data:
-                raise oefmt(space.w_IndexError, "pop from empty bytearray")
-            raise oefmt(space.w_IndexError, "pop index out of range")
+        if self._len() == 0:
+            raise oefmt(space.w_IndexError, "pop from empty bytearray")
+        index = self._fixindex(space, index, "pop index out of range")
+        result = self._data.pop(index)
         return space.newint(ord(result))
 
     def descr_remove(self, space, w_char):
         char = space.int_w(space.index(w_char))
-        try:
-            self.data.remove(chr(char))
-        except ValueError:
-            raise oefmt(space.w_ValueError, "value not found in bytearray")
+        _data = self._data
+        for index in range(self._offset, len(_data)):
+            if ord(_data[index]) == char:
+                del _data[index]
+                return
+        raise oefmt(space.w_ValueError, "value not found in bytearray")
 
     _StringMethods_descr_contains = descr_contains
     def descr_contains(self, space, w_sub):
         if space.isinstance_w(w_sub, space.w_int):
             char = space.int_w(w_sub)
-            return _descr_contains_bytearray(self.data, space, char)
+            return _descr_contains_bytearray(self.getdata(), space, char)
 
         return self._StringMethods_descr_contains(space, w_sub)
 
     def descr_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return self._new(self.data + w_other.data)
+            return self._new(self.getdata() + w_other.getdata())
 
         if isinstance(w_other, W_BytesObject):
             return self._add(self._op_val(space, w_other))
@@ -458,11 +500,37 @@
 
     @specialize.argtype(1)
     def _add(self, other):
-        return self._new(self.data + [other[i] for i in range(len(other))])
+        return self._new(self.getdata() + [other[i] for i in range(len(other))])
 
     def descr_reverse(self, space):
-        self.data.reverse()
+        self.getdata().reverse()
 
+    def descr_alloc(self, space):
+        return space.newint(len(self._data) + 1)   # includes the _offset part
+
+    def _convert_idx_params(self, space, w_start, w_end):
+        # optimization: this version doesn't force getdata()
+        start, end = unwrap_start_stop(space, self._len(), w_start, w_end)
+        ofs = self._offset
+        return (self._data, start + ofs, end + ofs, ofs)
+
+    def descr_getitem(self, space, w_index):
+        # optimization: this version doesn't force getdata()
+        if isinstance(w_index, W_SliceObject):
+            start, stop, step, sl = w_index.indices4(space, self._len())
+            if sl == 0:
+                return self._empty()
+            elif step == 1:
+                assert start >= 0 and stop >= 0
+                ofs = self._offset
+                return self._new(self._data[start + ofs : stop + ofs])
+            else:
+                start += self._offset
+                ret = _descr_getslice_slowpath(self._data, start, step, sl)
+                return self._new_from_list(ret)
+
+        index = space.getindex_w(w_index, space.w_IndexError, self._KIND1)
+        return self._getitem_result(space, index)
 
 
 # ____________________________________________________________
@@ -1133,6 +1201,8 @@
                         doc=BytearrayDocstrings.remove.__doc__),
     reverse = interp2app(W_BytearrayObject.descr_reverse,
                          doc=BytearrayDocstrings.reverse.__doc__),
+    __alloc__ = interp2app(W_BytearrayObject.descr_alloc,
+                           doc=BytearrayDocstrings.__alloc__.__doc__),
 )
 W_BytearrayObject.typedef.flag_sequence_bug_compat = True
 
@@ -1197,21 +1267,6 @@
                     "attempt to assign sequence of size %d to extended slice "
                     "of size %d", len2, slicelength)
 
-    if sequence2 is items:
-        if step > 0:
-            # Always copy starting from the right to avoid
-            # having to make a shallow copy in the case where
-            # the source and destination lists are the same list.
-            i = len2 - 1
-            start += i*step
-            while i >= 0:
-                items[start] = sequence2[i]
-                start -= step
-                i -= 1
-            return
-        else:
-            # Make a shallow copy to more easily handle the reversal case
-            sequence2 = list(sequence2)
     for i in range(len2):
         items[start] = sequence2[i]
         start += step
@@ -1220,36 +1275,47 @@
 class BytearrayBuffer(Buffer):
     _immutable_ = True
 
-    def __init__(self, data, readonly):
-        self.data = data
+    def __init__(self, ba, readonly=False):
+        self.ba = ba     # the W_BytearrayObject
         self.readonly = readonly
 
     def getlength(self):
-        return len(self.data)
+        return self.ba._len()
 
     def getitem(self, index):
-        return self.data[index]
+        ba = self.ba
+        return ba._data[ba._offset + index]
 
     def setitem(self, index, char):
-        self.data[index] = char
+        ba = self.ba
+        ba._data[ba._offset + index] = char
 
     def getslice(self, start, stop, step, size):
         if size == 0:
             return ""
         if step == 1:
             assert 0 <= start <= stop
-            if start == 0 and stop == len(self.data):
-                return "".join(self.data)
-            return "".join(self.data[start:stop])
+            ba = self.ba
+            start += ba._offset
+            stop += ba._offset
+            data = ba._data
+            if start != 0 or stop != len(data):
+                data = data[start:stop]
+            return "".join(data)
         return Buffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
+        ba = self.ba
+        start += ba._offset
         for i in range(len(string)):
-            self.data[start + i] = string[i]
+            ba._data[start + i] = string[i]
 
     def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
+        ba = self.ba
+        p = nonmoving_raw_ptr_for_resizable_list(ba._data)
+        p = rffi.ptradd(p, ba._offset)
+        return p
 
 
 @specialize.argtype(1)
@@ -1261,3 +1327,9 @@
         if selfvalue[i] > buffer[i]:
             return 1
     return 0
+
+def _tweak_for_tests(w_bytearray):
+    "Patched in test_bytearray.py"
+
+def _shrink_after_delete_from_start(w_bytearray):
+    w_bytearray.getdata()
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -26,7 +26,8 @@
         value = self._val(space)
         lenself = len(value)
         start, end = unwrap_start_stop(space, lenself, w_start, w_end)
-        return (value, start, end)
+        # the None means "no offset"; see bytearrayobject.py
+        return (value, start, end, None)
 
     def _multi_chr(self, c):
         return c
@@ -38,18 +39,18 @@
     #    pass
 
     def descr_contains(self, space, w_sub):
-        value = self._val(space)
+        value, start, end, _ = self._convert_idx_params(space, None, None)
         if self._use_rstr_ops(space, w_sub):
             other = self._op_val(space, w_sub)
-            return space.newbool(value.find(other) >= 0)
+            return space.newbool(value.find(other, start, end) >= 0)
 
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytesObject):
             other = self._op_val(space, w_sub)
-            res = find(value, other, 0, len(value))
+            res = find(value, other, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
-            res = find(value, buffer, 0, len(value))
+            res = find(value, buffer, start, end)
 
         return space.newbool(res >= 0)
 
@@ -146,7 +147,7 @@
         return self._new(centered)
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
-        value, start, end = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             return space.newint(value.count(self._op_val(space, w_sub), start,
@@ -155,7 +156,7 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = count(value, w_sub.data, start, end)
+            res = count(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = count(value, w_sub._value, start, end)
         else:
@@ -235,7 +236,7 @@
         return distance
 
     def descr_find(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             res = value.find(self._op_val(space, w_sub), start, end)
@@ -244,17 +245,18 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = find(value, w_sub.data, start, end)
+            res = find(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = find(value, w_sub._value, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
             res = find(value, buffer, start, end)
-
+        if ofs is not None and res >= 0:
+            res -= ofs
         return space.newint(res)
 
     def descr_rfind(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         if self._use_rstr_ops(space, w_sub):
             res = value.rfind(self._op_val(space, w_sub), start, end)
@@ -263,24 +265,25 @@
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if isinstance(w_sub, W_BytearrayObject):
-            res = rfind(value, w_sub.data, start, end)
+            res = rfind(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = rfind(value, w_sub._value, start, end)
         else:
             buffer = _get_buffer(space, w_sub)
             res = rfind(value, buffer, start, end)
-
+        if ofs is not None and res >= 0:
+            res -= ofs
         return space.newint(res)
 
     def descr_index(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if self._use_rstr_ops(space, w_sub):
             res = value.find(self._op_val(space, w_sub), start, end)
         elif isinstance(w_sub, W_BytearrayObject):
-            res = find(value, w_sub.data, start, end)
+            res = find(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = find(value, w_sub._value, start, end)
         else:
@@ -290,17 +293,19 @@
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.index")
+        if ofs is not None:
+            res -= ofs
         return space.newint(res)
 
     def descr_rindex(self, space, w_sub, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, ofs = self._convert_idx_params(space, w_start, w_end)
 
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
         from pypy.objspace.std.bytesobject import W_BytesObject
         if self._use_rstr_ops(space, w_sub):
             res = value.rfind(self._op_val(space, w_sub), start, end)
         elif isinstance(w_sub, W_BytearrayObject):
-            res = rfind(value, w_sub.data, start, end)
+            res = rfind(value, w_sub.getdata(), start, end)
         elif isinstance(w_sub, W_BytesObject):
             res = rfind(value, w_sub._value, start, end)
         else:
@@ -310,6 +315,8 @@
         if res < 0:
             raise oefmt(space.w_ValueError,
                         "substring not found in string.rindex")
+        if ofs is not None:
+            res -= ofs
         return space.newint(res)
 
     @specialize.arg(2)
@@ -612,7 +619,7 @@
         return self._newlist_unwrapped(space, strs)
 
     def descr_startswith(self, space, w_prefix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_prefix, space.w_tuple):
             return self._startswith_tuple(space, value, w_prefix, start, end)
         return space.newbool(self._startswith(space, value, w_prefix, start,
@@ -635,7 +642,7 @@
                          # bytearrays, but overridden for unicodes
 
     def descr_endswith(self, space, w_suffix, w_start=None, w_end=None):
-        (value, start, end) = self._convert_idx_params(space, w_start, w_end)
+        value, start, end, _ = self._convert_idx_params(space, w_start, w_end)
         if space.isinstance_w(w_suffix, space.w_tuple):
             return self._endswith_tuple(space, value, w_suffix, start, end)
         return space.newbool(self._endswith(space, value, w_suffix, start,
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -1,9 +1,26 @@
+import random
 from pypy import conftest
+from pypy.objspace.std import bytearrayobject
+
+class DontAccess(object):
+    pass
+dont_access = DontAccess()
+
 
 
 class AppTestBytesArray:
     def setup_class(cls):
         cls.w_runappdirect = cls.space.wrap(conftest.option.runappdirect)
+        def tweak(w_bytearray):
+            n = random.randint(-3, 16)
+            if n > 0:
+                w_bytearray._data = [dont_access] * n + w_bytearray._data
+                w_bytearray._offset += n
+        cls._old_tweak = [bytearrayobject._tweak_for_tests]
+        bytearrayobject._tweak_for_tests = tweak
+
+    def teardown_class(cls):
+        [bytearrayobject._tweak_for_tests] = cls._old_tweak
 
     def test_basics(self):
         b = bytearray()
@@ -67,6 +84,7 @@
         raises(IndexError, b.__getitem__, 4)
         assert b[1:5] == bytearray('est')
         assert b[slice(1,5)] == bytearray('est')
+        assert b[1:5:2] == bytearray(b'et')
 
     def test_arithmetic(self):
         b1 = bytearray('hello ')
@@ -204,6 +222,10 @@
         assert bytearray('ab').endswith(bytearray(''), 2) is True
         assert bytearray('ab').endswith(bytearray(''), 3) is False
 
+    def test_startswith_self(self):
+        b = bytearray(b'abcd')
+        assert b.startswith(b)
+
     def test_stringlike_conversions(self):
         # methods that should return bytearray (and not str)
         def check(result, expected):
@@ -330,6 +352,20 @@
         b.reverse()
         assert b == bytearray('olleh')
 
+    def test_delitem_from_front(self):
+        b = bytearray(b'abcdefghij')
+        del b[0]
+        del b[0]
+        assert len(b) == 8
+        assert b == bytearray(b'cdefghij')
+        del b[-8]
+        del b[-7]
+        assert len(b) == 6
+        assert b == bytearray(b'efghij')
+        del b[:3]
+        assert len(b) == 3
+        assert b == bytearray(b'hij')
+
     def test_delitem(self):
         b = bytearray('abc')
         del b[1]
@@ -412,6 +448,18 @@
         raises(TypeError, b.extend, [object()])
         raises(TypeError, b.extend, u"unicode")
 
+    def test_setitem_from_front(self):
+        b = bytearray(b'abcdefghij')
+        b[:2] = b''
+        assert len(b) == 8
+        assert b == bytearray(b'cdefghij')
+        b[:3] = b'X'
+        assert len(b) == 6
+        assert b == bytearray(b'Xfghij')
+        b[:2] = b'ABC'
+        assert len(b) == 7
+        assert b == bytearray(b'ABCghij')
+
     def test_setslice(self):
         b = bytearray('hello')
         b[:] = [ord(c) for c in 'world']
@@ -502,3 +550,78 @@
     def test_split_whitespace(self):
         b = bytearray(b'\x09\x0A\x0B\x0C\x0D\x1C\x1D\x1E\x1F')
         assert b.split() == [b'\x1c\x1d\x1e\x1f']
+
+    def test_dont_force_offset(self):
+        def make(x=b'abcdefghij', shift=3):
+            b = bytearray(b'?'*shift + x)
+            b + b''                       # force 'b'
+            del b[:shift]                 # add shift to b._offset
+            return b
+        assert make(shift=0).__alloc__() == 11
+        #
+        x = make(shift=3)
+        assert x.__alloc__() == 14
+        assert memoryview(x)[1] == 'b'
+        assert x.__alloc__() == 14
+        assert len(x) == 10
+        assert x.__alloc__() == 14
+        assert x[3] == ord('d')
+        assert x[-3] == ord('h')
+        assert x.__alloc__() == 14
+        assert x[3:-3] == b'defg'
+        assert x[-3:3:-1] == b'hgfe'
+        assert x.__alloc__() == 14
+        assert repr(x) == "bytearray(b'abcdefghij')"
+        assert x.__alloc__() == 14
+        #
+        x = make(shift=3)
+        x[3] = ord('D')
+        assert x.__alloc__() == 14
+        x[4:6] = b'EF'
+        assert x.__alloc__() == 14
+        x[6:8] = b'G'
+        assert x.__alloc__() == 13
+        x[-2:4:-2] = b'*/'
+        assert x.__alloc__() == 13
+        assert x == bytearray(b'abcDE/G*j')
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        assert len(x) == 26
+        assert x.__alloc__() == 38
+        del x[:1]
+        assert len(x) == 25
+        assert x.__alloc__() == 38
+        del x[0:5]
+        assert len(x) == 20
+        assert x.__alloc__() == 38
+        del x[0]
+        assert len(x) == 19
+        assert x.__alloc__() == 38
+        del x[0]                      # too much emptiness, forces now
+        assert len(x) == 18
+        assert x.__alloc__() == 19
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        del x[:9]                     # too much emptiness, forces now
+        assert len(x) == 17
+        assert x.__alloc__() == 18
+        #
+        x = make(b'abcdefghijklmnopqrstuvwxyz', shift=11)
+        assert x.__alloc__() == 38
+        del x[1]
+        assert x.__alloc__() == 37      # not forced, but the list shrank
+        del x[3:10:2]
+        assert x.__alloc__() == 33
+        assert x == bytearray(b'acdfhjlmnopqrstuvwxyz')
+        #
+        x = make(shift=3)
+        assert b'f' in x
+        assert b'ef' in x
+        assert b'efx' not in x
+        assert b'very long string longer than the original' not in x
+        assert x.__alloc__() == 14
+        assert x.find(b'f') == 5
+        assert x.rfind(b'f', 2, 11) == 5
+        assert x.find(b'fe') == -1
+        assert x.index(b'f', 2, 11) == 5
+        assert x.__alloc__() == 14

From pypy.commits at gmail.com  Sat May  6 14:51:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 06 May 2017 11:51:54 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Fix bytearray
Message-ID: <590e1b4a.db17370a.8c53d.5d4e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91195:dc72a659d99a
Date: 2017-05-06 19:50 +0100
http://bitbucket.org/pypy/pypy/changeset/dc72a659d99a/

Log:	Fix bytearray

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -2,18 +2,19 @@
 
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
 from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
         nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
+from rpython.rlib.buffer import Buffer
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.buffer import SimpleView
 from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
 from pypy.objspace.std.stringmethods import _descr_getslice_slowpath
@@ -46,7 +47,7 @@
                            ''.join(self._data[self._offset:]))
 
     def buffer_w(self, space, flags):
-        return BytearrayBuffer(self)
+        return SimpleView(BytearrayBuffer(self))
 
     def readbuf_w(self, space):
         return BytearrayBuffer(self, readonly=True)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -812,4 +812,4 @@
     return [selfvalue[start + i*step] for i in range(sl)]
 
 def _get_buffer(space, w_obj):
-    return space.buffer_w(w_obj, space.BUF_SIMPLE)
+    return space.buffer_w(w_obj, space.BUF_SIMPLE).as_readbuf()

From pypy.commits at gmail.com  Sat May  6 17:19:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 06 May 2017 14:19:46 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Add space.byte_w()
Message-ID: <590e3df2.743ced0a.e0b10.42c6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91196:97f60ebf3ec0
Date: 2017-05-06 20:42 +0100
http://bitbucket.org/pypy/pypy/changeset/97f60ebf3ec0/

Log:	Add space.byte_w()

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1654,6 +1654,23 @@
     def fsencode_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
 
+    def byte_w(self, w_obj):
+        """
+        Convert an index-like object to an interp-level char
+
+        Used for app-level code like "bytearray(b'abc')[0] = 42".
+        """
+        if self.isinstance_w(w_obj, self.w_bytes):
+            string = self.bytes_w(w_obj)
+            if len(string) != 1:
+                raise oefmt(self.w_ValueError, "string must be of size 1")
+            return string[0]
+        value = self.getindex_w(w_obj, None)
+        if not 0 <= value < 256:
+            # this includes the OverflowError in case the long is too large
+            raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
+        return chr(value)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -415,7 +415,7 @@
         else:
             idx = space.getindex_w(w_index, space.w_IndexError,
                                    "bytearray index")
-            newvalue = getbytevalue(space, w_other)
+            newvalue = space.byte_w(w_other)
             self._data[self._fixindex(space, idx)] = newvalue
 
     def descr_delitem(self, space, w_idx):
@@ -442,7 +442,7 @@
                              _shrink_after_delete_from_start, self)
 
     def descr_append(self, space, w_item):
-        self._data.append(getbytevalue(space, w_item))
+        self._data.append(space.byte_w(w_item))
 
     def descr_extend(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
@@ -452,10 +452,9 @@
 
     def descr_insert(self, space, w_idx, w_other):
         where = space.int_w(w_idx)
-        val = getbytevalue(space, w_other)
         data = self.getdata()
-        length = len(data)
-        index = get_positive_index(where, length)
+        index = get_positive_index(where, len(data))
+        val = space.byte_w(w_other)
         data.insert(index, val)
 
     @unwrap_spec(w_idx=WrappedDefault(-1))
@@ -552,20 +551,6 @@
 # ____________________________________________________________
 
 
-def getbytevalue(space, w_value):
-    if space.isinstance_w(w_value, space.w_bytes):
-        string = space.bytes_w(w_value)
-        if len(string) != 1:
-            raise oefmt(space.w_ValueError, "string must be of size 1")
-        return string[0]
-
-    value = space.getindex_w(w_value, None)
-    if not 0 <= value < 256:
-        # this includes the OverflowError in case the long is too large
-        raise oefmt(space.w_ValueError, "byte must be in range(0, 256)")
-    return chr(value)
-
-
 def new_bytearray(space, w_bytearraytype, data):
     w_obj = space.allocate_instance(W_BytearrayObject, w_bytearraytype)
     W_BytearrayObject.__init__(w_obj, data)
@@ -594,8 +579,7 @@
             if not e.match(space, space.w_StopIteration):
                 raise
             break
-        value = getbytevalue(space, w_item)
-        data.append(value)
+        data.append(space.byte_w(w_item))
         extended += 1
     if extended < length_hint:
         resizelist_hint(data, extended)
diff --git a/pypy/objspace/std/test/test_memoryobject.py b/pypy/objspace/std/test/test_memoryobject.py
--- a/pypy/objspace/std/test/test_memoryobject.py
+++ b/pypy/objspace/std/test/test_memoryobject.py
@@ -27,7 +27,8 @@
         v[0:3] = v[2:5]
         assert data == bytearray(eval("b'23f3fg'"))
         exc = raises(ValueError, "v[2] = 'spam'")
-        assert str(exc.value) == "cannot modify size of memoryview object"
+        assert str(exc.value) in ("cannot modify size of memoryview object",
+                                  "string must be of size 1")
         exc = raises(NotImplementedError, "v[0:2:2] = 'spam'")
         assert str(exc.value) == ""
 

From pypy.commits at gmail.com  Sun May  7 14:16:57 2017
From: pypy.commits at gmail.com (tobweber)
Date: Sun, 07 May 2017 11:16:57 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Implement a
 per thread adaptive single thread mode
Message-ID: <590f6499.a121ed0a.963b1.d92e@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2058:cbb625d908bf
Date: 2017-05-07 20:16 +0200
http://bitbucket.org/pypy/stmgc/changeset/cbb625d908bf/

Log:	Implement a per thread adaptive single thread mode

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1167,9 +1167,8 @@
     }
     if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
                                  in abort_data_structures_from_segment_num() */
-        stm_update_transaction_length();
         STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
-                                     stm_fill_mark_nursery_bytes);
+                                        stm_get_transaction_length(tl));
     }
     return repeat_count;
 }
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -23,61 +23,42 @@
 uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
 // uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.000001f)
-static float stm_relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
-static int stm_increase_transaction_length_backoff = 0;
+#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001f)
 
-static void reset_or_decrease_backoff(bool reset) {
-    int actual_backoff = stm_increase_transaction_length_backoff;
-    int expected_backoff;
-    int value = 5;
-    do {
-        expected_backoff = actual_backoff;
-        if (!reset) {
-            value = actual_backoff - 1;
-        }
-        actual_backoff = __atomic_exchange_n(
-            &stm_increase_transaction_length_backoff, value, __ATOMIC_RELAXED);
-    } while (expected_backoff != actual_backoff && (reset || actual_backoff > 0));
-}
-
-static float get_new_transaction_length(bool aborts, float previous) {
-    const int multiplier = 2;
+static float get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
+    const int multiplier = 100;
+    float previous = tl->relative_transaction_length;
     float new = previous;
     if (aborts) {
-        reset_or_decrease_backoff(true); // reset backoff
+        tl->transaction_length_backoff = 3;
         if (previous > STM_MIN_RELATIVE_TRANSACTION_LENGTH) {
             new = previous / multiplier;
         } else {
             new = 0;
         }
-    } else if (stm_increase_transaction_length_backoff == 0) {
+    } else if (tl->transaction_length_backoff == 0) {
         if (previous - (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1) < 0) {
             new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
         } else if (previous < 1) {
             new = previous * multiplier;
         }
     } else { // not abort and backoff != 0
-        reset_or_decrease_backoff(false); // decrease backoff by one
+        tl->transaction_length_backoff -= 1;
     }
     return new;
 }
 
 static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) {
-    float actual = stm_relative_transaction_length;
-    float expected;
-    do {
-        expected = actual;
-        float new = get_new_transaction_length(aborts, actual);
-        __atomic_exchange(
-            &stm_relative_transaction_length, &new, &actual, __ATOMIC_RELAXED);
-    } while (fabs(actual - expected) > (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1));
+    if (!tl->initialized) {
+        tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
+        tl->initialized = true;
+    }
+    float new = get_new_transaction_length(tl, aborts);
+    tl->relative_transaction_length = new;
 }
 
-static void stm_update_transaction_length(void) {
-    float relative_additional_length = stm_relative_transaction_length;
-    stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES +
-        (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
+static uintptr_t stm_get_transaction_length(stm_thread_local_t *tl) {
+    float relative_additional_length = tl->relative_transaction_length;
     if (timing_enabled()) {
         struct timespec relative_length = {
             .tv_sec = (int)relative_additional_length,
@@ -89,6 +70,8 @@
             STM_SINGLE_THREAD_MODE_ADAPTIVE,
             &stm_duration_payload);
     }
+    return DEFAULT_FILL_MARK_NURSERY_BYTES +
+        (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
 }
 
 
diff --git a/c8/stm/nursery.h b/c8/stm/nursery.h
--- a/c8/stm/nursery.h
+++ b/c8/stm/nursery.h
@@ -60,6 +60,6 @@
 static uint32_t stm_global_conflicts;
 
 static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts);
-static void stm_update_transaction_length(void);
+static uintptr_t stm_get_transaction_length(stm_thread_local_t *tl);
 
 #endif
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -9,6 +9,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
+#include <stdbool.h>
 #include <assert.h>
 #include <limits.h>
 #include <unistd.h>
@@ -88,6 +89,10 @@
     struct stm_thread_local_s *prev, *next;
     intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
+    /* adaptive single thread mode */
+    float relative_transaction_length;
+    int transaction_length_backoff;
+    bool initialized;
 } stm_thread_local_t;
 
 

From pypy.commits at gmail.com  Mon May  8 05:03:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 08 May 2017 02:03:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: The issue is even more specific than
 documented
Message-ID: <5910347f.879d370a.bb47d.3379@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91197:6bfa05038883
Date: 2017-05-08 11:03 +0200
http://bitbucket.org/pypy/pypy/changeset/6bfa05038883/

Log:	The issue is even more specific than documented

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -502,8 +502,10 @@
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
 * In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
-  even if the called function takes ``**kwargs``. E.g. this code always
-  produces a ``TypeError``, no matter what ``f`` is: ``f(**{1: 2})``.
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.
 
 * PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``

From pypy.commits at gmail.com  Mon May  8 06:01:31 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 08 May 2017 03:01:31 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: experimental checkin,
 reviews are welcome :). Refactor rgc._ResizableListSupportingRawPtr
 to use a proper lltype representation of its items,
 instead of a custom raw-malloced array. This makes it possible to implement
 ll_list(), i.e. the equivalent of llstr but for lists
Message-ID: <591041fb.d125ed0a.3c1e6.2e03@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91198:8a582e197e5b
Date: 2017-05-08 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/8a582e197e5b/

Log:	experimental checkin, reviews are welcome :). Refactor
	rgc._ResizableListSupportingRawPtr to use a proper lltype
	representation of its items, instead of a custom raw-malloced array.
	This makes it possible to implement ll_list(), i.e. the equivalent
	of llstr but for lists

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1051,17 +1051,25 @@
     rgc.nonmoving_raw_ptr_for_resizable_list() might be
     used if needed.  For now, only supports lists of chars.
     """
-    __slots__ = ('_raw_items',)   # either None or a rffi.CCHARP
+    __slots__ = ('_ll_list',)   # either None or a struct of TYPE=_get_lltype()
+
+    _LLTYPE = None
+    @classmethod
+    def _get_lltype(cls):
+        from rpython.rtyper.lltypesystem.rlist import make_LIST
+        if cls._LLTYPE is None:
+            cls._LLTYPE = make_LIST(lltype.Char)
+        return cls._LLTYPE
 
     def __init__(self, lst):
-        self._raw_items = None
+        self._ll_list = None
         self.__from_list(lst)
 
     def __resize(self):
         """Called before an operation changes the size of the list"""
-        if self._raw_items is not None:
+        if self._ll_list is not None:
             list.__init__(self, self.__as_list())
-            self._raw_items = None
+            self._ll_list = None
 
     def __from_list(self, lst):
         """Initialize the list from a copy of the list 'lst'."""
@@ -1072,39 +1080,39 @@
             return
         if len(self) != len(lst):
             self.__resize()
-        if self._raw_items is None:
+        if self._ll_list is None:
             list.__init__(self, lst)
         else:
-            assert len(self) == self._raw_items._obj.getlength() == len(lst)
+            assert len(self) == self._ll_list.length == len(lst)
             for i in range(len(self)):
-                self._raw_items[i] = lst[i]
+                self._ll_list.items[i] = lst[i]
 
     def __as_list(self):
         """Return a list (the same or a different one) which contains the
         items in the regular way."""
-        if self._raw_items is None:
+        if self._ll_list is None:
             return self
-        length = self._raw_items._obj.getlength()
+        length = self._ll_list.length
         assert length == len(self)
-        return [self._raw_items[i] for i in range(length)]
+        return [self._ll_list.items[i] for i in range(length)]
 
     def __getitem__(self, index):
-        if self._raw_items is None:
+        if self._ll_list is None:
             return list.__getitem__(self, index)
         if index < 0:
             index += len(self)
         if not (0 <= index < len(self)):
             raise IndexError
-        return self._raw_items[index]
+        return self._ll_list.items[index]
 
     def __setitem__(self, index, new):
-        if self._raw_items is None:
+        if self._ll_list is None:
             return list.__setitem__(self, index, new)
         if index < 0:
             index += len(self)
         if not (0 <= index < len(self)):
             raise IndexError
-        self._raw_items[index] = new
+        self._ll_list.items[index] = new
 
     def __delitem__(self, index):
         self.__resize()
@@ -1219,14 +1227,18 @@
         self.__from_list(lst)
 
     def _nonmoving_raw_ptr_for_resizable_list(self):
-        if self._raw_items is None:
+        if self._ll_list is None:
             existing_items = list(self)
             from rpython.rtyper.lltypesystem import lltype, rffi
-            self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self),
-                                           flavor='raw', immortal=True)
+            LIST = self._get_lltype()
+            n = len(self)
+            self._ll_list = lltype.malloc(LIST, immortal=True)
+            self._ll_list.length = n
+            self._ll_list.items = lltype.malloc(LIST.items.TO, n)
             self.__from_list(existing_items)
-            assert self._raw_items is not None
-        return self._raw_items
+            assert self._ll_list is not None
+        return ll_nonmovable_raw_ptr_for_resizable_list(self._ll_list)
+        #return self._raw_items
 
 def resizable_list_supporting_raw_ptr(lst):
     return _ResizableListSupportingRawPtr(lst)
@@ -1256,6 +1268,9 @@
         return s_list
 
     def specialize_call(self, hop):
+        if hop.args_r[0].LIST != _ResizableListSupportingRawPtr._get_lltype():
+            raise ValueError('Resizable list of chars does not have the '
+                             'expected low-level type')
         hop.exception_cannot_occur()
         return hop.inputarg(hop.args_r[0], 0)
 
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -351,14 +351,14 @@
 
     p = lst
     check_nonresizing()
-    assert lst._raw_items is None
-    lst._nonmoving_raw_ptr_for_resizable_list()
-    p = lst._raw_items
+    assert lst._ll_list is None
+    p = lst._nonmoving_raw_ptr_for_resizable_list()
+    ll_list = lst._ll_list
     check_nonresizing()
-    assert lst._raw_items == p
-    assert p[0] == 'a'
-    assert p[1] == 'b'
-    assert p[2] == 'c'
+    assert lst._ll_list == ll_list
+    assert p[0] == ll_list.items[0] == 'a'
+    assert p[1] == ll_list.items[1] == 'b'
+    assert p[2] == ll_list.items[2] == 'c'
 
     def do_resizing_operation():
         del lst[1]
@@ -397,7 +397,7 @@
     assert lst == ['a', 'b', 'c']
     for expect in do_resizing_operation():
         assert lst == expect
-        assert lst._raw_items is None
+        assert lst._ll_list is None
         lst = ['a', 'b', 'c']
         lst = rgc.resizable_list_supporting_raw_ptr(lst)
         lst._nonmoving_raw_ptr_for_resizable_list()
diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py
--- a/rpython/rtyper/lltypesystem/rlist.py
+++ b/rpython/rtyper/lltypesystem/rlist.py
@@ -4,7 +4,7 @@
 from rpython.rtyper.error import TyperError
 from rpython.rtyper.lltypesystem import rstr
 from rpython.rtyper.lltypesystem.lltype import (GcForwardReference, Ptr, GcArray,
-     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod)
+     GcStruct, Void, Signed, malloc, typeOf, nullptr, typeMethod, Char)
 from rpython.rtyper.rlist import (AbstractBaseListRepr, AbstractListRepr,
     AbstractFixedSizeListRepr, AbstractListIteratorRepr, ll_setitem_nonneg,
     ADTIList, ADTIFixedList, dum_nocheck)
@@ -30,6 +30,18 @@
 #    item_t list_items[]
 #
 
+def make_LIST(ITEMTYPE):
+    """
+    Return the low-level type for a resizable list of ITEMTYPE
+    """
+    from rpython.rtyper.lltypesystem.rstr import CharRepr
+    assert ITEMTYPE is Char, 'only Char is supported for now'
+    # XXX: maybe we should think of a better way to build the type?
+    list_of_char_repr = ListRepr(None, CharRepr())
+    list_of_char_repr._setup_repr()
+    return list_of_char_repr.LIST
+
+
 class BaseListRepr(AbstractBaseListRepr):
     rstr_ll = rstr.LLHelpers
 

From pypy.commits at gmail.com  Mon May  8 06:01:33 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 08 May 2017 03:01:33 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement
 rgc.ll_for_resizable_list, which is the equivalent of llstr(), but for lists
Message-ID: <591041fd.0c4c370a.12401.1672@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91199:ed23f8739819
Date: 2017-05-08 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/ed23f8739819/

Log:	implement rgc.ll_for_resizable_list, which is the equivalent of
	llstr(), but for lists

diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1226,7 +1226,7 @@
         list.sort(lst, *args, **kwds)
         self.__from_list(lst)
 
-    def _nonmoving_raw_ptr_for_resizable_list(self):
+    def _get_ll_list(self):
         if self._ll_list is None:
             existing_items = list(self)
             from rpython.rtyper.lltypesystem import lltype, rffi
@@ -1237,8 +1237,11 @@
             self._ll_list.items = lltype.malloc(LIST.items.TO, n)
             self.__from_list(existing_items)
             assert self._ll_list is not None
-        return ll_nonmovable_raw_ptr_for_resizable_list(self._ll_list)
-        #return self._raw_items
+        return self._ll_list
+
+    def _nonmoving_raw_ptr_for_resizable_list(self):
+        ll_list = self._get_ll_list()
+        return ll_nonmovable_raw_ptr_for_resizable_list(ll_list)
 
 def resizable_list_supporting_raw_ptr(lst):
     return _ResizableListSupportingRawPtr(lst)
@@ -1247,6 +1250,18 @@
     assert isinstance(lst, _ResizableListSupportingRawPtr)
     return lst._nonmoving_raw_ptr_for_resizable_list()
 
+def ll_for_resizable_list(lst):
+    """
+    This is the equivalent of llstr(), but for lists. It can be called only if
+    the list has been created by calling resizable_list_supporting_raw_ptr().
+
+    In theory, all the operations on lst are immediately visible also on
+    ll_list. However, support for that is incomplete in
+    _ResizableListSupportingRawPtr and as such, the pointer becomes invalid as
+    soon as you call a resizing operation on lst.
+    """
+    assert isinstance(lst, _ResizableListSupportingRawPtr)
+    return lst._get_ll_list()
 
 def _check_resizable_list_of_chars(s_list):
     from rpython.annotator import model as annmodel
@@ -1289,6 +1304,23 @@
         return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list,
                                  v_list)
 
+class Entry(ExtRegistryEntry):
+    _about_ = ll_for_resizable_list
+
+    def compute_result_annotation(self, s_list):
+        from rpython.rtyper.llannotation import lltype_to_annotation
+        _check_resizable_list_of_chars(s_list)
+        LIST = _ResizableListSupportingRawPtr._get_lltype()
+        return lltype_to_annotation(lltype.Ptr(LIST))
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        assert hop.args_r[0].lowleveltype == hop.r_result.lowleveltype
+        v_ll_list, = hop.inputargs(*hop.args_r)
+        return hop.genop('same_as', [v_ll_list],
+                         resulttype = hop.r_result.lowleveltype)
+
+
 @jit.dont_look_inside
 def ll_nonmovable_raw_ptr_for_resizable_list(ll_list):
     """
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -304,6 +304,50 @@
     data = subprocess.check_output([str(exename), '.', '.', '.'])
     assert data.strip().endswith('OK!')
 
+def test_ll_for_resizable_list():
+    def f(n):
+        lst = ['a', 'b', 'c']
+        lst = rgc.resizable_list_supporting_raw_ptr(lst)
+        lst.append(chr(n))
+        assert lst[3] == chr(n)
+        assert lst[-1] == chr(n)
+        #
+        ll_list = rgc.ll_for_resizable_list(lst)
+        assert lst[:] == ['a', 'b', 'c', chr(n)]
+        assert ll_list.length == 4
+        assert [ll_list.items[i] for i in range(4)] == ['a', 'b', 'c', chr(n)]
+        #
+        lst[-3] = 'X'
+        assert ll_list.items[1] == 'X'
+        ll_list.items[2] = 'Y'
+        assert lst[-2] == 'Y'
+        #
+        return lst
+    #
+    # direct untranslated run
+    lst = f(35)
+    assert isinstance(lst, rgc._ResizableListSupportingRawPtr)
+    #
+    # llinterp run
+    interpret(f, [35])
+    #
+    # compilation with the GC transformer
+    import subprocess
+    from rpython.translator.interactive import Translation
+    #
+    def main(argv):
+        f(len(argv))
+        print "OK!"
+        return 0
+    #
+    t = Translation(main, gc="incminimark")
+    t.disable(['backendopt'])
+    t.set_backend_extra_options(c_debug_defines=True)
+    exename = t.compile()
+    data = subprocess.check_output([str(exename), '.', '.', '.'])
+    assert data.strip().endswith('OK!')
+
+
 def test_ListSupportingRawPtr_direct():
     lst = ['a', 'b', 'c']
     lst = rgc.resizable_list_supporting_raw_ptr(lst)
@@ -353,7 +397,8 @@
     check_nonresizing()
     assert lst._ll_list is None
     p = lst._nonmoving_raw_ptr_for_resizable_list()
-    ll_list = lst._ll_list
+    ll_list = rgc.ll_for_resizable_list(lst)
+    assert ll_list is lst._ll_list
     check_nonresizing()
     assert lst._ll_list == ll_list
     assert p[0] == ll_list.items[0] == 'a'

From pypy.commits at gmail.com  Mon May  8 06:16:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 08 May 2017 03:16:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add another note
Message-ID: <59104599.8e3eed0a.18746.6082@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91200:73b4ce214eb8
Date: 2017-05-08 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/73b4ce214eb8/

Log:	Add another note

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -505,7 +505,8 @@
   even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
   keys in these two cases (and only there, as far as we know).  E.g. this
   code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
-  ``dict(**{1: 2})``.
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
 
 * PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``

From pypy.commits at gmail.com  Mon May  8 09:18:38 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 08 May 2017 06:18:38 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: whoo,
 implement BytearrayBuffer.type_read using ll_for_resizable_list and
 llops.gc_load_index
Message-ID: <5910702e.a121ed0a.963b1.63d4@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91201:50c818fb6f86
Date: 2017-05-08 15:18 +0200
http://bitbucket.org/pypy/pypy/changeset/50c818fb6f86/

Log:	whoo, implement BytearrayBuffer.type_read using
	ll_for_resizable_list and llops.gc_load_index

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -7,7 +7,8 @@
 from rpython.rlib.debug import check_list_of_chars
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-        nonmoving_raw_ptr_for_resizable_list)
+                              nonmoving_raw_ptr_for_resizable_list,
+                              ll_for_resizable_list)
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -1253,15 +1254,17 @@
 
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
-        # XXX: this is sub-efficient, because it forces the bytearray to
-        # become non-movable in order to do the raw_load. In theory, it should
-        # be possible to do the same using llop.gc_load_indexed, the same way
-        # we do it for strings. However, we cannot do it because there is no
-        # way to convert self.data from being a high-level list into the ll
-        # equivalent.
+        from rpython.rtyper.lltypesystem import lltype, llmemory
         from rpython.rtyper.lltypesystem.lloperation import llop
-        raw_ptr = self.get_raw_address()
-        return llop.raw_load(TP, raw_ptr, byte_offset)
+        from rpython.rlib.rgc import get_LIST_OF_CHAR
+        LIST = get_LIST_OF_CHAR()
+        ll_data = ll_for_resizable_list(self.data)
+        ll_items = ll_data.items
+        base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
+        scale_factor = llmemory.sizeof(lltype.Char)
+        return llop.gc_load_indexed(TP, ll_items, byte_offset,
+                                    scale_factor, base_ofs)
+
 
 
 @specialize.argtype(1)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1263,6 +1263,10 @@
     assert isinstance(lst, _ResizableListSupportingRawPtr)
     return lst._get_ll_list()
 
+ at specialize.memo()
+def get_LIST_OF_CHAR():
+    return _ResizableListSupportingRawPtr._get_lltype()
+
 def _check_resizable_list_of_chars(s_list):
     from rpython.annotator import model as annmodel
     from rpython.rlib import debug

From pypy.commits at gmail.com  Mon May  8 11:51:16 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 May 2017 08:51:16 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Backport cpyext changes
Message-ID: <591093f4.2622ed0a.232ef.1bd4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91202:302a031b3560
Date: 2017-05-08 16:50 +0100
http://bitbucket.org/pypy/pypy/changeset/302a031b3560/

Log:	Backport cpyext changes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -221,7 +221,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
 
@@ -233,7 +234,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -245,7 +247,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -254,7 +257,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
 
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -41,15 +41,15 @@
     assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
     view = py_obj.c_view
-    ndim = w_obj.buf.getndim()
+    ndim = w_obj.getndim()
     if ndim >= Py_MAX_NDIMS:
         # XXX warn?
         return
-    fill_Py_buffer(space, w_obj.buf, view)
+    fill_Py_buffer(space, w_obj.view, view)
     try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
         view.c_obj = make_ref(space, w_userdata)
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
         view.c_obj = make_ref(space, w_s)
@@ -95,7 +95,6 @@
     mem_obj.c_view.c_obj = rffi.cast(PyObject, 0)
     _dealloc(space, py_obj)
 
-
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -19,8 +19,10 @@
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
+
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
@@ -322,7 +324,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(Buffer):
+class CPyBuffer(BufferView):
     # Similar to Py_buffer
     _immutable_ = True
 
@@ -333,9 +335,12 @@
         self.space = space
         self.ptr = ptr
         self.size = size
-        self.w_obj = w_obj # kept alive
+        self.w_obj = w_obj  # kept alive
         self.pyobj = as_pyobj(space, w_obj)
         self.format = format
+        self.ndim = ndim
+        self.itemsize = itemsize
+
         if not shape:
             self.shape = [size]
         else:
@@ -344,8 +349,6 @@
             self.strides = [1]
         else:
             self.strides = strides
-        self.ndim = ndim
-        self.itemsize = itemsize
         self.readonly = readonly
         self.needs_decref = needs_decref
         self.releasebufferproc = releasebufferproc
@@ -378,8 +381,20 @@
     def getlength(self):
         return self.size
 
-    def getitem(self, index):
-        return self.ptr[index]
+    def getbytes(self, start, size):
+        return ''.join([self.ptr[i] for i in range(start, start + size)])
+
+    def setbytes(self, start, string):
+        # absolutely no safety checks, what could go wrong?
+        for i in range(len(string)):
+            self.ptr[start + i] = string[i]
+
+    def as_readbuf(self):
+        return CBuffer(self)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return CBuffer(self)
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
@@ -399,10 +414,6 @@
     def getndim(self):
         return self.ndim
 
-    def setitem(self, index, char):
-        # absolutely no safety checks, what could go wrong?
-        self.ptr[index] = char
-
 class FQ(rgc.FinalizerQueue):
     Class = CPyBuffer
     def finalizer_trigger(self):
@@ -414,6 +425,37 @@
 
 fq = FQ()
 
+
+class CBuffer(Buffer):
+    _immutable_ = True
+    def __init__(self, view):
+        self.view = view
+        self.readonly = view.readonly
+
+    def getlength(self):
+        return self.view.getlength()
+
+    def getitem(self, index):
+        return self.view.ptr[index]
+
+    def getslice(self, start, stop, step, size):
+        assert step == 1
+        assert stop - start == size
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), start)
+        return rffi.charpsize2str(ptr, size)
+
+    def setitem(self, index, char):
+        self.view.ptr[index] = char
+
+    def setslice(self, index, s):
+        assert s is not None
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), index)
+        rffi.str2chararray(s, ptr, len(s))
+
+    def get_raw_address(self):
+        return cts.cast('char *', self.view.ptr)
+
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
@@ -430,7 +472,7 @@
         buf = CPyBuffer(space, ptr[0], size, w_self,
                                releasebufferproc=rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return buf.wrap(space)
 
 def wrap_getwritebuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -448,7 +490,7 @@
         buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
                                releasebufferproc=rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return buf.wrap(space)
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
@@ -490,7 +532,7 @@
                             needs_decref=True,
                             releasebufferproc = rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return buf.wrap(space)
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -667,6 +709,7 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         slot_func = slot_tp_getattro
+
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -744,24 +787,24 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         args = Arguments(space, [space.newint(flags)])
         w_obj = space.call_args(space.get(buff_fn, w_self), args)
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
@@ -771,23 +814,23 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         w_obj = w_self
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,6 +344,7 @@
 #endif
     if (m == NULL)
         INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -111,7 +111,7 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyLong_FromLong(-1);
+                    return NULL;
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
                 return PyLong_FromLong(view->len / view->itemsize);

From pypy.commits at gmail.com  Mon May  8 13:34:22 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 May 2017 10:34:22 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: _io and writebuf_w changes
Message-ID: <5910ac1e.0950370a.a1b9d.67d3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91203:4eabf29be4e9
Date: 2017-05-08 18:33 +0100
http://bitbucket.org/pypy/pypy/changeset/4eabf29be4e9/

Log:	_io and writebuf_w changes

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,9 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.buffer import StringBuffer
 
+from pypy.interpreter.buffer import BufferInterfaceNotFound
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -396,9 +398,6 @@
 class DescrMismatch(Exception):
     pass
 
-class BufferInterfaceNotFound(Exception):
-    pass
-
 @specialize.memo()
 def wrappable_class_name(Class):
     try:
@@ -1513,10 +1512,15 @@
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+        except OperationError:
+            self._getarg_error("read-write buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.writebuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a writeable buffer object")
+            self._getarg_error("read-write buffer", w_obj)
 
     def charbuf_w(self, w_obj):
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
@@ -1562,16 +1566,7 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
-            try:
-                return w_obj.buffer_w(self, self.BUF_WRITABLE)
-            except OperationError:
-                self._getarg_error("read-write buffer", w_obj)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.writebuf_w(self)
-            except BufferInterfaceNotFound:
-                self._getarg_error("read-write buffer", w_obj)
+            return self.writebuf_w(w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -1,12 +1,15 @@
 from __future__ import with_statement
 
+from rpython.rlib.signature import signature
+from rpython.rlib import types
+
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rgc import (
-    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import SimpleView
+
+from rpython.rlib.buffer import ByteBuffer, SubBuffer
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib import rposix
@@ -16,7 +19,6 @@
     check_readable_w, check_writable_w, check_seekable_w)
 from pypy.module._io.interp_io import W_BlockingIOError
 from rpython.rlib import rthread
-from rpython.rtyper.lltypesystem import rffi
 
 STATE_ZERO, STATE_OK, STATE_DETACHED = range(3)
 
@@ -88,12 +90,16 @@
         self._unsupportedoperation(space, "detach")
 
     def readinto_w(self, space, w_buffer):
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        return self._readinto(space, w_buffer, "read")
+
+    def _readinto(self, space, w_buffer, methodname):
+        rwbuffer = space.writebuf_w(w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, "read", space.newint(length))
+        w_data = space.call_method(self, methodname, space.newint(length))
 
         if not space.isinstance_w(w_data, space.w_bytes):
-            raise oefmt(space.w_TypeError, "read() should return bytes")
+            raise oefmt(space.w_TypeError, "%s() should return bytes",
+                        methodname)
         data = space.bytes_w(w_data)
         rwbuffer.setslice(0, data)
         return space.newint(len(data))
@@ -108,25 +114,6 @@
     readinto = interp2app(W_BufferedIOBase.readinto_w),
 )
 
-class RawBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, buf, start, length):
-        self.buf = buf
-        self.start = start
-        self.length = length
-        self.readonly = False
-
-    def getlength(self):
-        return self.length
-
-    def setitem(self, index, char):
-        self.buf[self.start + index] = char
-
-    def get_raw_address(self):
-        ptr = nonmoving_raw_ptr_for_resizable_list(self.buf)
-        return rffi.ptradd(ptr, self.start)
-
 class BufferedMixin:
     _mixin_ = True
 
@@ -165,8 +152,7 @@
             raise oefmt(space.w_ValueError,
                         "buffer size must be strictly positive")
 
-        self.buffer = resizable_list_supporting_raw_ptr(['\0'] *
-                                                        self.buffer_size)
+        self.buffer = ByteBuffer(self.buffer_size)
 
         self.lock = TryLock(space)
 
@@ -238,6 +224,7 @@
 
     # ______________________________________________
 
+    @signature(types.any(), returns=types.int())
     def _readahead(self):
         if self.readable and self.read_end != -1:
             available = self.read_end - self.pos
@@ -278,7 +265,7 @@
                 else:
                     offset = pos
                 if -self.pos <= offset <= available:
-                    newpos = self.pos + offset
+                    newpos = self.pos + int(offset)
                     assert newpos >= 0
                     self.pos = newpos
                     return space.newint(current - available + offset)
@@ -374,11 +361,7 @@
         return written
 
     def _raw_write(self, space, start, end):
-        # XXX inefficient
-        l = []
-        for i in range(start, end):
-            l.append(self.buffer[i])
-        return self._write(space, ''.join(l))
+        return self._write(space, self.buffer[start:end])
 
     def detach_w(self, space):
         self._check_init(space)
@@ -428,6 +411,7 @@
     @unwrap_spec(size=int)
     def peek_w(self, space, size=0):
         self._check_init(space)
+        self._check_closed(space, "peek of closed file")
         with self.lock:
             if self.writable:
                 self._flush_and_rewind_unlocked(space)
@@ -439,7 +423,7 @@
             # buffer.
             have = self._readahead()
             if have > 0:
-                data = ''.join(self.buffer[self.pos:self.pos+have])
+                data = self.buffer[self.pos:self.pos+have]
                 return space.newbytes(data)
 
             # Fill the buffer from the raw stream, and copy it to the result
@@ -449,7 +433,7 @@
             except BlockingIOError:
                 size = 0
             self.pos = 0
-            data = ''.join(self.buffer[:size])
+            data = self.buffer[0:size]
             return space.newbytes(data)
 
     @unwrap_spec(size=int)
@@ -486,7 +470,7 @@
             if size > have:
                 size = have
             endpos = self.pos + size
-            data = ''.join(self.buffer[self.pos:endpos])
+            data = self.buffer[self.pos:endpos]
             self.pos = endpos
             return space.newbytes(data)
 
@@ -498,7 +482,7 @@
         current_size = self._readahead()
         data = None
         if current_size:
-            data = ''.join(self.buffer[self.pos:self.pos + current_size])
+            data = self.buffer[self.pos:self.pos + current_size]
             builder.append(data)
             self.pos += current_size
         # We're going past the buffer's bounds, flush it
@@ -524,11 +508,13 @@
         return space.newbytes(builder.build())
 
     def _raw_read(self, space, buffer, start, length):
+        assert buffer is not None
         length = intmask(length)
-        w_buf = space.newbuffer(RawBuffer(buffer, start, length))
+        start = intmask(start)
+        w_view = SimpleView(SubBuffer(buffer, start, length)).wrap(space)
         while True:
             try:
-                w_size = space.call_method(self.w_raw, "readinto", w_buf)
+                w_size = space.call_method(self.w_raw, "readinto", w_view)
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue  # try again
@@ -565,12 +551,12 @@
         if n <= current_size:
             return self._read_fast(n)
 
-        result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n)
+        result_buffer = ByteBuffer(n)
         remaining = n
         written = 0
         if current_size:
-            for i in range(current_size):
-                result_buffer[written + i] = self.buffer[self.pos + i]
+            result_buffer.setslice(
+                written, self.buffer[self.pos:self.pos + current_size])
             remaining -= current_size
             written += current_size
             self.pos += current_size
@@ -592,7 +578,7 @@
                     return None
                 size = 0
             if size == 0:
-                return ''.join(result_buffer[:written])
+                return result_buffer[0:written]
             remaining -= size
             written += size
 
@@ -614,14 +600,13 @@
             if remaining > 0:
                 if size > remaining:
                     size = remaining
-                for i in range(size):
-                    result_buffer[written + i] = self.buffer[self.pos + i]
+                result_buffer.setslice(
+                    written, self.buffer[self.pos:self.pos + size])
                 self.pos += size
-
                 written += size
                 remaining -= size
 
-        return ''.join(result_buffer[:written])
+        return result_buffer[0:written]
 
     def _read_fast(self, n):
         """Read n bytes from the buffer if it can, otherwise return None.
@@ -629,7 +614,7 @@
         current_size = self._readahead()
         if n <= current_size:
             endpos = self.pos + n
-            res = ''.join(self.buffer[self.pos:endpos])
+            res = self.buffer[self.pos:endpos]
             self.pos = endpos
             return res
         return None
@@ -652,11 +637,11 @@
         else:
             pos = -1
         if pos >= 0:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1]))
+            w_res = space.newbytes(self.buffer[self.pos:pos+1])
             self.pos = pos + 1
             return w_res
         if have == limit:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have]))
+            w_res = space.newbytes(self.buffer[self.pos:self.pos+have])
             self.pos += have
             return w_res
 
@@ -665,7 +650,7 @@
             # Now we try to get some more from the raw stream
             chunks = []
             if have > 0:
-                chunks.extend(self.buffer[self.pos:self.pos+have])
+                chunks.append(self.buffer[self.pos:self.pos+have])
                 written += have
                 self.pos += have
                 if limit >= 0:
@@ -683,13 +668,14 @@
                 pos = 0
                 found = False
                 while pos < have:
-                    c = self.buffer[pos]
+                    # 'buffer.data[]' instead of 'buffer[]' because RPython...
+                    c = self.buffer.data[pos]
                     pos += 1
                     if c == '\n':
                         self.pos = pos
                         found = True
                         break
-                chunks.extend(self.buffer[0:pos])
+                chunks.append(self.buffer[0:pos])
                 if found:
                     break
                 if have == limit:
@@ -716,7 +702,6 @@
         size = len(data)
 
         with self.lock:
-
             if (not (self.readable and self.read_end != -1) and
                 not (self.writable and self.write_end != -1)):
                 self.pos = 0
@@ -746,7 +731,8 @@
                     self._reader_reset_buf()
                 # Make some place by shifting the buffer
                 for i in range(self.write_pos, self.write_end):
-                    self.buffer[i - self.write_pos] = self.buffer[i]
+                    # XXX: messing with buffer internals
+                    self.buffer.data[i - self.write_pos] = self.buffer.data[i]
                 self.write_end -= self.write_pos
                 self.raw_pos -= self.write_pos
                 newpos = self.pos - self.write_pos
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -34,17 +34,17 @@
         size = convert_size(space, w_size)
         return space.newbytes(self.read(size))
 
+    def read1_w(self, space, w_size):
+        return self.read_w(space, w_size)
+
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
         return space.newbytes(self.readline(limit))
 
-    def read1_w(self, space, w_size):
-        return self.read_w(space, w_size)
-
     def readinto_w(self, space, w_buffer):
         self._check_closed(space)
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        rwbuffer = space.writebuf_w(w_buffer)
         size = rwbuffer.getlength()
 
         output = self.read(size)
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -1,9 +1,4 @@
 from rpython.annotator.model import SomeInstance, s_None
-from pypy.interpreter import argument, gateway
-from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.objspace.std.sliceobject import W_SliceObject
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.objectmodel import (instantiate, we_are_translated, specialize,
     not_rpython)
 from rpython.rlib.nonconst import NonConstant
@@ -14,6 +9,13 @@
 from rpython.tool.sourcetools import compile2, func_with_new_name
 from rpython.translator.translator import TranslationContext
 
+from pypy.tool.option import make_config
+from pypy.interpreter import argument, gateway
+from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
+from pypy.interpreter.buffer import StringBuffer, SimpleView
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.objspace.std.sliceobject import W_SliceObject
+
 
 class W_MyObject(W_Root):
     typedef = None
@@ -41,7 +43,7 @@
         is_root(w_subtype)
 
     def buffer_w(self, space, flags):
-        return StringBuffer("foobar")
+        return SimpleView(StringBuffer("foobar"))
 
     def str_w(self, space):
         return NonConstant("foobar")
@@ -123,7 +125,7 @@
 
 BUILTIN_TYPES = ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict',
                  'unicode', 'complex', 'slice', 'bool', 'basestring', 'object',
-                 'bytearray', 'buffer', 'set', 'frozenset']
+                 'set', 'frozenset', 'bytearray', 'buffer', 'memoryview']
 
 INTERP_TYPES = ['function', 'builtin_function', 'module', 'getset_descriptor',
                 'instance', 'classobj']
@@ -197,7 +199,7 @@
     def newseqiter(self, x):
         return w_some_obj()
 
-    def newbuffer(self, x):
+    def newmemoryview(self, x):
         return w_some_obj()
 
     @not_rpython

From pypy.commits at gmail.com  Mon May  8 14:14:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 08 May 2017 11:14:30 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: progress
Message-ID: <5910b586.1824ed0a.9c604.29cc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91204:4f1fa0d61a99
Date: 2017-05-08 19:13 +0100
http://bitbucket.org/pypy/pypy/changeset/4f1fa0d61a99/

Log:	progress

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1504,10 +1504,15 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+        except OperationError:
+            self._getarg_error("convertible to a buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.readbuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a readable buffer object")
+            self._getarg_error("convertible to a buffer", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
@@ -1549,12 +1554,10 @@
             if self.isinstance_w(w_obj, self.w_unicode):
                 return self.str(w_obj).readbuf_w(self)
             try:
-                return w_obj.buffer_w(self, 0)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.readbuf_w(self)
-            except BufferInterfaceNotFound:
+                return self.readbuf_w(w_obj)
+            except OperationError as e:
+                if not e.match(self, self.w_TypeError):
+                    raise
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_bytes):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -4,6 +4,7 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 from pypy.objspace.std.bufferobject import W_Buffer
+from pypy.interpreter.buffer import SimpleView
 
 from rpython.rlib.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
@@ -60,7 +61,7 @@
         if space.isinstance_w(w_other, space.w_unicode):
             return space.w_NotImplemented
         try:
-            other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+            other_buf = space.readbuf_w(w_other)
         except OperationError as e:
             if e.async(space):
                 raise
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,24 +105,10 @@
 # ____________________________________________________________
 
 def _fetch_as_read_buffer(space, w_x):
-    # xxx do we really need to implement the same mess as in CPython 2.7
-    # w.r.t. buffers and memoryviews??
-    try:
-        buf = space.readbuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_SIMPLE)
-    return buf
+    return space.readbuf_w(w_x)
 
 def _fetch_as_write_buffer(space, w_x):
-    try:
-        buf = space.writebuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_WRITABLE)
-    return buf
+    return space.writebuf_w(w_x)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def from_buffer(space, w_ctype, w_x):
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,5 +1,6 @@
+from rpython.rtyper.lltypesystem import rffi
+
 from rpython.rlib.buffer import Buffer
-from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -1,5 +1,6 @@
 import sys
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
@@ -371,13 +372,7 @@
         self._ll_buffer = self.ll_buffer
 
     def buffer_w(self, space, flags):
-        return RawFFIBuffer(self)
-
-    def readbuf_w(self, space):
-        return RawFFIBuffer(self)
-
-    def writebuf_w(self, space):
-        return RawFFIBuffer(self)
+        return SimpleView(RawFFIBuffer(self))
 
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -382,6 +382,7 @@
         raises(self.struct.error, self.struct.unpack, "i", b)
 
     def test_pack_unpack_buffer(self):
+        import sys
         import array
         b = array.array('c', '\x00' * 19)
         sz = self.struct.calcsize("ii")
@@ -391,9 +392,11 @@
                                       self.struct.pack("ii", 17, 42) +
                                       '\x00' * (19-sz-2))
         exc = raises(TypeError, self.struct.pack_into, "ii", buffer(b), 0, 17, 42)
-        assert str(exc.value) == "must be read-write buffer, not buffer"
+        if '__pypy__' in sys.modules:
+            assert str(exc.value) == "must be read-write buffer, not buffer"
         exc = raises(TypeError, self.struct.pack_into, "ii", 'test', 0, 17, 42)
-        assert str(exc.value) == "must be read-write buffer, not str"
+        if '__pypy__' in sys.modules:
+            assert str(exc.value) == "must be read-write buffer, not str"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
--- a/pypy/objspace/std/strbufobject.py
+++ b/pypy/objspace/std/strbufobject.py
@@ -2,9 +2,9 @@
 
 import py
 
-from pypy.objspace.std.bytesobject import (W_AbstractBytesObject,
-    W_BytesObject, StringBuffer)
+from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
 from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import SimpleView, StringBuffer
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rstring import StringBuilder
 
@@ -38,10 +38,7 @@
         return self.force()
 
     def buffer_w(self, space, flags):
-        return StringBuffer(self.force())
-
-    def readbuf_w(self, space):
-        return StringBuffer(self.force())
+        return SimpleView(StringBuffer(self.force()))
 
     def descr_len(self, space):
         return space.newint(self.length)

From pypy.commits at gmail.com  Mon May  8 14:44:50 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 08 May 2017 11:44:50 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: 1) avoid to take an unneeded
 string slice to skip the chars after typed_read;
 2) as it was written before, we did the out-of-bound checks *after* the call
 to typed_read, which means that we risked to read past the end of the
 buffer
Message-ID: <5910bca2.11a2370a.eb860.8cc1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91205:5155ad0ff2ca
Date: 2017-05-08 20:44 +0200
http://bitbucket.org/pypy/pypy/changeset/5155ad0ff2ca/

Log:	1) avoid to take an unneeded string slice to skip the chars after
	typed_read; 2) as it was written before, we did the out-of-bound
	checks *after* the call to typed_read, which means that we risked to
	read past the end of the buffer

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -140,13 +140,20 @@
         if self.pos != self.length:
             raise StructError("unpack str size too long for format")
 
+    def can_advance(self, count):
+        end = self.pos + count
+        return end <= self.length
+
+    def advance(self, count):
+        if not self.can_advance(count):
+            raise StructError("unpack str size too short for format")
+        self.pos += count
+
     def read(self, count):
-        end = self.pos + count
-        if end > self.length:
-            raise StructError("unpack str size too short for format")
-        s = self.buf.getslice(self.pos, end, 1, count)
-        self.pos = end
-        return s
+        curpos = self.pos
+        end = curpos + count
+        self.advance(count) # raise if we are out of bound
+        return self.buf.getslice(curpos, end, 1, count)
 
     @specialize.argtype(1)
     def appendobj(self, value):
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -16,14 +16,20 @@
         self.length = len(s)
         self.inputpos = 0
 
+    def can_advance(self, count):
+        end = self.inputpos + count
+        return end <= self.length
+
+    def advance(self, count):
+        if not self.can_advance(count):
+            raise StructError("unpack str size too short for format")
+        self.inputpos += count
+
     def read(self, count):
-        end = self.inputpos + count
-        if end > self.length:
-            raise StructError("unpack str size too short for format")
-        size = end - self.inputpos
-        s = self.inputbuf.getslice(self.inputpos, end, 1, size)
-        self.inputpos = end
-        return s
+        curpos = self.inputpos
+        end = curpos + count
+        self.advance(count) # raise if we are out of bound
+        return self.inputbuf.getslice(curpos, end, 1, count)
 
     def align(self, mask):
         self.inputpos = (self.inputpos + mask) & ~mask
@@ -46,8 +52,11 @@
         def get_buffer_and_pos(self):
             return self.mr.inputbuf, self.mr.inputpos
 
-        def skip(self, size):
-            self.read(size) # XXX, could avoid taking the slice
+        def can_advance(self, size):
+            return self.mr.can_advance(size)
+
+        def advance(self, size):
+            self.mr.advance(size)
     ReaderForPos.__name__ = 'ReaderForPos%d' % pos
     return ReaderForPos
 
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -149,11 +149,17 @@
             # buf.typed_read to raise CannotRead in case it is not aligned
             # *and* it is not supported.
             raise CannotRead
-        # we need to call skip *after* we typed_read(), because if it raises
-        # we do not want to skip
-        result = buf.typed_read(TYPE, pos)
-        fmtiter.skip(size)
-        return result
+        #
+        # typed_read does not do any bound check, so we must call it only if
+        # we are sure there are at least "size" bytes to read
+        if fmtiter.can_advance(size):
+            result = buf.typed_read(TYPE, pos)
+            fmtiter.advance(size)
+            return result
+        else:
+            # this will raise StructError
+            fmtiter.advance(size)
+            assert False, 'fmtiter.advance should have raised!'
     return do_unpack_fastpath
 
 @specialize.argtype(0)
diff --git a/rpython/rlib/rstruct/test/test_runpack.py b/rpython/rlib/rstruct/test/test_runpack.py
--- a/rpython/rlib/rstruct/test/test_runpack.py
+++ b/rpython/rlib/rstruct/test/test_runpack.py
@@ -1,6 +1,8 @@
+import pytest
 from rpython.rtyper.test.tool import BaseRtypingTest
 from rpython.rlib.rstruct.runpack import runpack
 from rpython.rlib.rstruct import standardfmttable
+from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.rarithmetic import LONG_BIT
 import struct
 
@@ -22,6 +24,18 @@
         assert fn() == 124
         assert self.interpret(fn, []) == 124
 
+    def test_unpack_error(self):
+        data = '123' # 'i' expects 4 bytes, not 3
+        def fn():
+            try:
+                runpack('i', data)
+            except StructError:
+                return True
+            else:
+                return False
+        assert fn()
+        assert self.interpret(fn, [])
+
     def test_unpack_single(self):
         data = struct.pack('i', 123)
         def fn():

From pypy.commits at gmail.com  Mon May  8 14:58:14 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 08 May 2017 11:58:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: update prebuilt external dependencies
 for win32
Message-ID: <5910bfc6.2823ed0a.f968c.a27a@mx.google.com>

Author: mattip <matti.picus at gmail.com>
Branch: 
Changeset: r91206:064a4230a254
Date: 2017-05-08 21:56 +0300
http://bitbucket.org/pypy/pypy/changeset/064a4230a254/

Log:	update prebuilt external dependencies for win32

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (for post-5.7.1 builds) with sha256 checksum 
-``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -128,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -236,6 +236,9 @@
     copy out32\*.lib <somewhere in LIB>
     xcopy /S include\openssl <somewhere in INCLUDE>
 
+For tests you will also need the dlls::
+    nmake -f ms\ntdll.mak install
+    copy out32dll\*.dll <somewhere in PATH>
 
 TkInter module support
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -245,18 +248,17 @@
 directory found for the release script, create the dlls, libs, headers and
 runtime by running::
 
-	svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
-	svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
-	cd tcl85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
-	nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
-	cd ..\..\tk85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+    svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+    svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+    cd tcl85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
+    nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+    cd ..\..\tk85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+    copy ..\..\tcltk\bin\* <somewhere in PATH>
+    copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+    xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
 
 The lzma compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -161,9 +161,9 @@
                 tktcldir = p.dirpath().join('..').join('lib')
                 shutil.copytree(str(tktcldir), str(pypydir.join('tcl')))
             except WindowsError:
-                print >>sys.stderr, """Packaging Tk runtime failed.
-tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib
-directory next to the dlls, as per build instructions."""
+                print >>sys.stderr, r"""Packaging Tk runtime failed.
+tk85.dll and tcl85.dll found in %s, expecting to find runtime in %s
+directory next to the dlls, as per build instructions.""" %(p, tktcldir)
                 import traceback;traceback.print_exc()
                 raise MissingDependenciesError('Tk runtime')
 

From pypy.commits at gmail.com  Tue May  9 10:18:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 07:18:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: polish things a bit: rename
 rlist.make_LIST into LIST_OF,
 and use it everywhere instead of the ugly
 _ResizableListSupportingRawPtr._get_lltype()
Message-ID: <5911cf99.c156370a.73c4f.0328@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91207:bc449c2983bb
Date: 2017-05-09 16:17 +0200
http://bitbucket.org/pypy/pypy/changeset/bc449c2983bb/

Log:	polish things a bit: rename rlist.make_LIST into LIST_OF, and use it
	everywhere instead of the ugly
	_ResizableListSupportingRawPtr._get_lltype()

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1256,10 +1256,9 @@
     def typed_read(self, TP, byte_offset):
         from rpython.rtyper.lltypesystem import lltype, llmemory
         from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rlib.rgc import get_LIST_OF_CHAR
-        LIST = get_LIST_OF_CHAR()
         ll_data = ll_for_resizable_list(self.data)
         ll_items = ll_data.items
+        LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
         base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
         scale_factor = llmemory.sizeof(lltype.Char)
         return llop.gc_load_indexed(TP, ll_items, byte_offset,
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1051,15 +1051,7 @@
     rgc.nonmoving_raw_ptr_for_resizable_list() might be
     used if needed.  For now, only supports lists of chars.
     """
-    __slots__ = ('_ll_list',)   # either None or a struct of TYPE=_get_lltype()
-
-    _LLTYPE = None
-    @classmethod
-    def _get_lltype(cls):
-        from rpython.rtyper.lltypesystem.rlist import make_LIST
-        if cls._LLTYPE is None:
-            cls._LLTYPE = make_LIST(lltype.Char)
-        return cls._LLTYPE
+    __slots__ = ('_ll_list',)   # either None or a struct of TYPE=LIST_OF(Char)
 
     def __init__(self, lst):
         self._ll_list = None
@@ -1227,10 +1219,11 @@
         self.__from_list(lst)
 
     def _get_ll_list(self):
+        from rpython.rtyper.lltypesystem import rffi
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
         if self._ll_list is None:
+            LIST = LIST_OF(lltype.Char)
             existing_items = list(self)
-            from rpython.rtyper.lltypesystem import lltype, rffi
-            LIST = self._get_lltype()
             n = len(self)
             self._ll_list = lltype.malloc(LIST, immortal=True)
             self._ll_list.length = n
@@ -1263,10 +1256,6 @@
     assert isinstance(lst, _ResizableListSupportingRawPtr)
     return lst._get_ll_list()
 
- at specialize.memo()
-def get_LIST_OF_CHAR():
-    return _ResizableListSupportingRawPtr._get_lltype()
-
 def _check_resizable_list_of_chars(s_list):
     from rpython.annotator import model as annmodel
     from rpython.rlib import debug
@@ -1287,7 +1276,8 @@
         return s_list
 
     def specialize_call(self, hop):
-        if hop.args_r[0].LIST != _ResizableListSupportingRawPtr._get_lltype():
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
+        if hop.args_r[0].LIST != LIST_OF(lltype.Char):
             raise ValueError('Resizable list of chars does not have the '
                              'expected low-level type')
         hop.exception_cannot_occur()
@@ -1312,9 +1302,10 @@
     _about_ = ll_for_resizable_list
 
     def compute_result_annotation(self, s_list):
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
         from rpython.rtyper.llannotation import lltype_to_annotation
         _check_resizable_list_of_chars(s_list)
-        LIST = _ResizableListSupportingRawPtr._get_lltype()
+        LIST = LIST_OF(lltype.Char)
         return lltype_to_annotation(lltype.Ptr(LIST))
 
     def specialize_call(self, hop):
diff --git a/rpython/rtyper/lltypesystem/rlist.py b/rpython/rtyper/lltypesystem/rlist.py
--- a/rpython/rtyper/lltypesystem/rlist.py
+++ b/rpython/rtyper/lltypesystem/rlist.py
@@ -30,16 +30,23 @@
 #    item_t list_items[]
 #
 
-def make_LIST(ITEMTYPE):
+def LIST_OF(ITEMTYPE, cache={}):
     """
     Return the low-level type for a resizable list of ITEMTYPE
     """
+    try:
+        return cache[ITEMTYPE]
+    except KeyError:
+        pass
+    
     from rpython.rtyper.lltypesystem.rstr import CharRepr
     assert ITEMTYPE is Char, 'only Char is supported for now'
     # XXX: maybe we should think of a better way to build the type?
     list_of_char_repr = ListRepr(None, CharRepr())
     list_of_char_repr._setup_repr()
-    return list_of_char_repr.LIST
+    LIST = list_of_char_repr.LIST
+    cache[ITEMTYPE] = LIST
+    return LIST
 
 
 class BaseListRepr(AbstractBaseListRepr):

From pypy.commits at gmail.com  Tue May  9 11:32:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 May 2017 08:32:44 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue 314
Message-ID: <5911e11c.c302370a.80fbc.1512@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2929:cc66a9e5ae92
Date: 2017-05-09 17:32 +0200
http://bitbucket.org/cffi/cffi/changeset/cc66a9e5ae92/

Log:	Issue 314

	Fix the line numbers by discovering that ``# NUMBER "LINE"`` is
	supported by pycparser and using it.

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -258,15 +258,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "<cdef source string>"')
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +282,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"<cdef source string>:(\d+):", msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -248,6 +248,16 @@
         );
       """))
 
+Note also that pycparser, the underlying C parser, recognizes
+preprocessor-like directives in the following format: ``# NUMBER
+"FILE"``.  For example, if you put ``# 42 "foo.h"`` in the middle of the
+string passed to ``cdef()`` and there is an error two lines later, then
+it is reported with an error message that starts with ``foo.h:43:`` (the
+line which is given the number 42 is the line immediately after the
+directive).  *New in version 1.11:*  CFFI automatically puts the line
+``# 1 "<cdef source string>"`` just before the string you give to
+``cdef()``.
+
 
 .. _`ffi.set_unicode()`:
 
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -234,7 +234,20 @@
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()

From pypy.commits at gmail.com  Tue May  9 12:07:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 May 2017 09:07:29 -0700 (PDT)
Subject: [pypy-commit] cffi default: Try to systematically include the line
 number
Message-ID: <5911e941.d5a0370a.f208c.1c3b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2930:a772dd1ab09f
Date: 2017-05-09 18:07 +0200
http://bitbucket.org/cffi/cffi/changeset/a772dd1ab09f/

Log:	Try to systematically include the line number

diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -266,7 +267,7 @@
                             ' __dotdotdot__;')
         # this forces pycparser to consider the following in the file
         # called <cdef source string> from line 1
-        csourcelines.append('# 1 "<cdef source string>"')
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
         fullcsource = '\n'.join(csourcelines)
         if lock is not None:
@@ -287,7 +288,7 @@
         # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        match = re.match(r"<cdef source string>:(\d+):", msg)
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
         if match:
             linenum = int(match.group(1), 10)
             csourcelines = csource.splitlines()
@@ -327,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -354,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/cffi/error.py b/cffi/error.py
--- a/cffi/error.py
+++ b/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -254,7 +254,7 @@
 string passed to ``cdef()`` and there is an error two lines later, then
 it is reported with an error message that starts with ``foo.h:43:`` (the
 line which is given the number 42 is the line immediately after the
-directive).  *New in version 1.11:*  CFFI automatically puts the line
+directive).  *New in version 1.10.1:*  CFFI automatically puts the line
 ``# 1 "<cdef source string>"`` just before the string you give to
 ``cdef()``.
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,15 @@
 ======================
 
 
+v1.10.1
+=======
+
+* Fixed the line numbers reported in case of ``cdef()`` errors.
+  Also, I just noticed, but pycparser always supported the preprocessor
+  directive ``# 42 "foo.h"`` to mean "from the next line, we're in file
+  foo.h starting from line 42", which it puts in the error messages.
+
+
 v1.10
 =====
 
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -228,8 +228,9 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
@@ -291,7 +292,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -449,3 +451,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')

From pypy.commits at gmail.com  Tue May  9 12:25:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 May 2017 09:25:56 -0700 (PDT)
Subject: [pypy-commit] cffi default: Print the current directory (or the
 value tmpdir if explicit)
Message-ID: <5911ed94.4fc3370a.98269.1e0b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2931:f1bd1d88c018
Date: 2017-05-09 18:25 +0200
http://bitbucket.org/cffi/cffi/changeset/f1bd1d88c018/

Log:	Print the current directory (or the value tmpdir if explicit) in
	ffi.compile(verbose=True).

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1479,6 +1479,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print '%s %r' % (msg, os.path.abspath(tmpdir))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)

From pypy.commits at gmail.com  Tue May  9 12:47:57 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 09:47:57 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add rlib.buffer.RawBuffer,
 which automatically implements typed_read in terms of
 get_raw_address
Message-ID: <5911f2bd.8e3eed0a.ec71d.223f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91208:f01cd8ba4297
Date: 2017-05-09 18:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f01cd8ba4297/

Log:	add rlib.buffer.RawBuffer, which automatically implements typed_read
	in terms of get_raw_address

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -92,6 +92,24 @@
         raise CannotRead
 
 
+class RawBuffer(Buffer):
+    """
+    A buffer which is baked by a raw, non-movable memory area. It implementes
+    typed_read in terms of get_raw_address()
+
+    NOTE: this assumes that get_raw_address() is cheap. Do not use this as a
+    base class if get_raw_address() is potentially costly, like for example if
+    you call rgc.nonmoving_raw_ptr_for_resizable_list
+    """
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        """
+        Read the value of type TP starting at byte_offset. No bounds checks
+        """
+        ptr = self.get_raw_address()
+        return llop.raw_load(TP, ptr, byte_offset)
+
 
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,11 +1,26 @@
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import r_singlefloat
-from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer
+from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer, RawBuffer
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 from rpython.rtyper.test.tool import BaseRtypingTest
 
+class MyRawBuffer(RawBuffer):
+
+    def __init__(self, data):
+        self._buf = lltype.malloc(rffi.CCHARP.TO, len(data), flavor='raw')
+        for i, ch in enumerate(data):
+            self._buf[i] = ch
+
+    def get_raw_address(self):
+        return self._buf
+
+    def __del__(self):
+        lltype.free(self._buf, flavor='raw')
+        self._buf = None
+
+
 def test_string_buffer():
     buf = StringBuffer('hello world')
     assert buf.getitem(4) == 'o'
@@ -97,6 +112,14 @@
         return subbuf.typed_read(TYPE, offset)
 
 
+class TestRawBufferTypedReadDirect(BaseTypedReadTest):
+
+    def read(self, TYPE, data, offset):
+        buf = MyRawBuffer(data)
+        return buf.typed_read(TYPE, offset)
+    
+
+
 class TestCompiled(BaseTypedReadTest):
     cache = {}
 

From pypy.commits at gmail.com  Tue May  9 12:48:00 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 09:48:00 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: use the new RawBuffer as a
 base class for ArrayBuffer: as a consequence,
 now it automatically gets the benefits of the fastpath in rstruct :)
Message-ID: <5911f2c0.0685370a.2c8e9.2101@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91209:a5ddf413daad
Date: 2017-05-09 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/a5ddf413daad/

Log:	use the new RawBuffer as a base class for ArrayBuffer: as a
	consequence, now it automatically gets the benefits of the fastpath
	in rstruct :)

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,5 @@
 from rpython.rlib import jit, rgc
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
@@ -796,7 +796,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayBuffer(Buffer):
+class ArrayBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, array, readonly):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -480,7 +480,7 @@
 
 
 class AppTestFastPath(object):
-    spaceconfig = dict(usemodules=['struct', '__pypy__'])
+    spaceconfig = dict(usemodules=['array', 'struct', '__pypy__'])
 
     def setup_class(cls):
         from rpython.rlib.rstruct import standardfmttable
@@ -509,6 +509,12 @@
         assert self.struct.unpack_from("ii", buf, offset) == (42, 43)
 
     def test_unpack_bytearray(self):
-        buf = self.struct.pack("iii", 0, 42, 43)
-        buf = bytearray(buf)
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = bytearray(data)
         assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_unpack_array(self):
+        import array
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = array.array('c', data)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)

From pypy.commits at gmail.com  Tue May  9 12:48:02 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 09:48:02 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: use RawBuffer for all the
 various raw-memory backed buffers we have
Message-ID: <5911f2c2.d438370a.6330d.25e7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91210:5bd9b59fb003
Date: 2017-05-09 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/5bd9b59fb003/

Log:	use RawBuffer for all the various raw-memory backed buffers we have

diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -5,13 +5,13 @@
 from pypy.module._cffi_backend import ctypestruct
 from pypy.objspace.std.bufferobject import W_Buffer
 
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(Buffer):
+class LLBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,10 +1,10 @@
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -21,7 +21,7 @@
 from pypy.module.cpyext import userslot
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
 from rpython.tool.sourcetools import func_renamer
@@ -322,7 +322,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(Buffer):
+class CPyBuffer(RawBuffer):
     # Similar to Py_buffer
     _immutable_ = True
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -2,7 +2,7 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -702,7 +702,7 @@
         free_raw_storage(self.storage)
 
 
-class ArrayBuffer(Buffer):
+class ArrayBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, impl, readonly):
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from rpython.rlib import rmmap, rarithmetic, objectmodel
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
 
@@ -330,7 +330,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(Buffer):
+class MMapBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):

From pypy.commits at gmail.com  Tue May  9 12:48:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 09 May 2017 09:48:47 -0700 (PDT)
Subject: [pypy-commit] cffi default: Py3 compat
Message-ID: <5911f2ef.1a2ec80a.fbc5b.26f8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2932:bac0dd1b86f9
Date: 2017-05-09 18:31 +0200
http://bitbucket.org/cffi/cffi/changeset/bac0dd1b86f9/

Log:	Py3 compat

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1484,7 +1484,7 @@
                         msg = 'the current directory is'
                     else:
                         msg = 'setting the current directory to'
-                    print '%s %r' % (msg, os.path.abspath(tmpdir))
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)

From pypy.commits at gmail.com  Tue May  9 13:13:48 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 10:13:48 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rpython fix
Message-ID: <5911f8cc.4220ed0a.7087d.28bf@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91211:402336bdaae0
Date: 2017-05-09 19:13 +0200
http://bitbucket.org/pypy/pypy/changeset/402336bdaae0/

Log:	rpython fix

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -840,7 +840,7 @@
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
                 self.array._charbuf_stop()
-        return Buffer.getslice(self, start, stop, step, size)
+        return RawBuffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
         return self.array._charbuf_start()
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -101,6 +101,7 @@
     base class if get_raw_address() is potentially costly, like for example if
     you call rgc.nonmoving_raw_ptr_for_resizable_list
     """
+    _immutable_ = True
 
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):

From pypy.commits at gmail.com  Tue May  9 13:20:29 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 10:20:29 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: more rpython fix
Message-ID: <5911fa5d.2536ed0a.15139.2874@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91212:a3879489f4b4
Date: 2017-05-09 19:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a3879489f4b4/

Log:	more rpython fix

diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -34,7 +34,7 @@
     def getslice(self, start, stop, step, size):
         if step == 1:
             return rffi.charpsize2str(rffi.ptradd(self.raw_cdata, start), size)
-        return Buffer.getslice(self, start, stop, step, size)
+        return RawBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         raw_cdata = rffi.ptradd(self.raw_cdata, start)
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -350,7 +350,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return Buffer.getslice(self, start, stop, step, size)
+            return RawBuffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()

From pypy.commits at gmail.com  Tue May  9 13:43:43 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 09 May 2017 10:43:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: rename harmless copy-paste
Message-ID: <5911ffcf.af25ed0a.bf3df.2a75@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91213:31566fa18b67
Date: 2017-05-09 20:31 +0300
http://bitbucket.org/pypy/pypy/changeset/31566fa18b67/

Log:	rename harmless copy-paste

diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -34,7 +34,7 @@
 cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct)
 
 @bootstrap_function
-def init_stringobject(space):
+def init_tupleobject(space):
     "Type description of PyTupleObject"
     make_typedescr(space.w_tuple.layout.typedef,
                    basestruct=PyTupleObject.TO,

From pypy.commits at gmail.com  Tue May  9 13:43:46 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 09 May 2017 10:43:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix table wrapping,
 from rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html
Message-ID: <5911ffd2.0a39ed0a.f62df.2e37@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91214:b48484eba9d0
Date: 2017-05-09 20:42 +0300
http://bitbucket.org/pypy/pypy/changeset/b48484eba9d0/

Log:	fix table wrapping, from rackerlabs.github.io/docs-rackspace/tools
	/rtd-tables.html

diff --git a/pypy/doc/_static/themes_override.css b/pypy/doc/_static/themes_override.css
new file mode 100644
--- /dev/null
+++ b/pypy/doc/_static/themes_override.css
@@ -0,0 +1,15 @@
+/* override table width restrictions */
+ at media screen and (min-width: 767px) {
+
+   .wy-table-responsive table td {
+      /* !important prevents the common CSS stylesheets from overriding
+         this as on RTD they are loaded after this stylesheet */
+      white-space: normal !important;
+   }
+
+   .wy-table-responsive {
+      overflow: visible !important;
+   }
+}
+
+
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.4'
+version = '5.7'
 # The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.7.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -141,7 +141,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-# html_static_path = ['_static']
+html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
@@ -181,6 +181,11 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'PyPydoc'
 
+html_context = {
+    'css_files': [
+        '_static/themes_override.css', # override too wide tables
+    ],
+}
 
 # -- Options for LaTeX output --------------------------------------------------
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -29,6 +29,14 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+CPython 3.5 compatible versions
+-------------------------------
+
+.. toctree::
+
+    whatsnew-pypy3-head.rst
+    whatsnew-pypy3-5.7.0.rst
+
 CPython 3.3 compatible versions
 -------------------------------
 
diff --git a/rpython/doc/_static/themes_override.css b/rpython/doc/_static/themes_override.css
new file mode 100644
--- /dev/null
+++ b/rpython/doc/_static/themes_override.css
@@ -0,0 +1,15 @@
+/* override table width restrictions */
+ at media screen and (min-width: 767px) {
+
+   .wy-table-responsive table td {
+      /* !important prevents the common CSS stylesheets from overriding
+         this as on RTD they are loaded after this stylesheet */
+      white-space: normal !important;
+   }
+
+   .wy-table-responsive {
+      overflow: visible !important;
+   }
+}
+
+
diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py
--- a/rpython/doc/conf.py
+++ b/rpython/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'RPython'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '4.0'
+version = '5.7'
 # The full version, including alpha/beta/rc tags.
-release = '4.0.0'
+release = '5.7.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -184,6 +184,11 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'RPythondoc'
 
+html_context = {
+    'css_files': [
+        '_static/themes_override.css', # override too wide tables
+    ],
+}
 
 # -- Options for LaTeX output --------------------------------------------------
 

From pypy.commits at gmail.com  Tue May  9 14:18:51 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 09 May 2017 11:18:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: back out b48484eba9d0,
 readthedocs is messed up
Message-ID: <5912080b.9724ed0a.3eb84.3641@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91215:90fb9edb78c5
Date: 2017-05-09 21:17 +0300
http://bitbucket.org/pypy/pypy/changeset/90fb9edb78c5/

Log:	back out b48484eba9d0, readthedocs is messed up

diff --git a/pypy/doc/_static/themes_override.css b/pypy/doc/_static/themes_override.css
deleted file mode 100644
--- a/pypy/doc/_static/themes_override.css
+++ /dev/null
@@ -1,15 +0,0 @@
-/* override table width restrictions */
- at media screen and (min-width: 767px) {
-
-   .wy-table-responsive table td {
-      /* !important prevents the common CSS stylesheets from overriding
-         this as on RTD they are loaded after this stylesheet */
-      white-space: normal !important;
-   }
-
-   .wy-table-responsive {
-      overflow: visible !important;
-   }
-}
-
-
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2017, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.7'
+version = '5.4'
 # The full version, including alpha/beta/rc tags.
-release = '5.7.1'
+release = '5.4.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -141,7 +141,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+# html_static_path = ['_static']
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
@@ -181,11 +181,6 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'PyPydoc'
 
-html_context = {
-    'css_files': [
-        '_static/themes_override.css', # override too wide tables
-    ],
-}
 
 # -- Options for LaTeX output --------------------------------------------------
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -29,14 +29,6 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
-CPython 3.5 compatible versions
--------------------------------
-
-.. toctree::
-
-    whatsnew-pypy3-head.rst
-    whatsnew-pypy3-5.7.0.rst
-
 CPython 3.3 compatible versions
 -------------------------------
 
diff --git a/rpython/doc/_static/themes_override.css b/rpython/doc/_static/themes_override.css
deleted file mode 100644
--- a/rpython/doc/_static/themes_override.css
+++ /dev/null
@@ -1,15 +0,0 @@
-/* override table width restrictions */
- at media screen and (min-width: 767px) {
-
-   .wy-table-responsive table td {
-      /* !important prevents the common CSS stylesheets from overriding
-         this as on RTD they are loaded after this stylesheet */
-      white-space: normal !important;
-   }
-
-   .wy-table-responsive {
-      overflow: visible !important;
-   }
-}
-
-
diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py
--- a/rpython/doc/conf.py
+++ b/rpython/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'RPython'
-copyright = u'2017, The PyPy Project'
+copyright = u'2016, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.7'
+version = '4.0'
 # The full version, including alpha/beta/rc tags.
-release = '5.7.1'
+release = '4.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -184,11 +184,6 @@
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'RPythondoc'
 
-html_context = {
-    'css_files': [
-        '_static/themes_override.css', # override too wide tables
-    ],
-}
 
 # -- Options for LaTeX output --------------------------------------------------
 

From pypy.commits at gmail.com  Tue May  9 15:45:15 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 12:45:15 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: kill rlib/strstorage.py since
 it is no longer needed: the only place where it was still used was in the
 slow-path of float/double unpacking in runpck.py: replace it with the
 equivalent functionality exposed by StringBuffer.typed_read
Message-ID: <59121c4b.e42ac80a.5e11a.4792@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91216:7a021235c44a
Date: 2017-05-09 21:44 +0200
http://bitbucket.org/pypy/pypy/changeset/7a021235c44a/

Log:	kill rlib/strstorage.py since it is no longer needed: the only place
	where it was still used was in the slow-path of float/double
	unpacking in runpck.py: replace it with the equivalent functionality
	exposed by StringBuffer.typed_read

diff --git a/rpython/jit/metainterp/test/test_strstorage.py b/rpython/jit/metainterp/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/metainterp/test/test_strstorage.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import py
-import sys
-import struct
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.strstorage import str_storage_getitem
-from rpython.rlib.test.test_strstorage import BaseStrStorageTest
-from rpython.jit.codewriter import longlong
-from rpython.jit.metainterp.history import getkind
-from rpython.jit.metainterp.test.support import LLJitMixin
-
-class TestStrStorage(BaseStrStorageTest, LLJitMixin):
-
-    # for the individual tests see
-    # ====> ../../../rlib/test/test_strstorage.py
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        def f():
-            return str_storage_getitem(TYPE, buf, offset)
-        res = self.interp_operations(f, [], supports_singlefloats=True)
-        #
-        kind = getkind(TYPE)[0] # 'i' or 'f'
-        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
-                                       'finish': 1})
-        #
-        if TYPE == lltype.SingleFloat:
-            # interp_operations returns the int version of r_singlefloat, but
-            # our tests expects to receive an r_singlefloat: let's convert it
-            # back!
-            return longlong.int2singlefloat(res)
-        return res
-
-    #def str_storage_supported(self, TYPE):
-    #    py.test.skip('this is not a JIT test')
-
-    def test_force_virtual_str_storage(self):
-        byteorder = sys.byteorder
-        size = rffi.sizeof(lltype.Signed)
-        def f(val):
-            if byteorder == 'little':
-                x = chr(val) + '\x00'*(size-1)
-            else:
-                x = '\x00'*(size-1) + chr(val)
-            return str_storage_getitem(lltype.Signed, x, 0)
-        res = self.interp_operations(f, [42], supports_singlefloats=True)
-        assert res == 42
-        self.check_operations_history({
-            'newstr': 1,              # str forcing
-            'strsetitem': 1,          # str forcing
-            'call_pure_r': 1,         # str forcing (copystrcontent)
-            'guard_no_exception': 1,  # str forcing
-            'gc_load_indexed_i': 1,   # str_storage_getitem
-            'finish': 1
-            })
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -12,7 +12,8 @@
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.strstorage import str_storage_getitem
+from rpython.rlib.buffer import StringBuffer
+#from rpython.rlib.strstorage import str_storage_getitem
 from rpython.rlib import rarithmetic
 from rpython.rlib.buffer import CannotRead
 from rpython.rtyper.lltypesystem import rffi
@@ -210,9 +211,12 @@
             # fast path
             val = unpack_fastpath(TYPE)(fmtiter)
         except CannotRead:
-            # slow path, take the slice
+            # slow path: we should arrive here only if we could not unpack
+            # because of alignment issues. So we copy the slice into a new
+            # string, which is guaranteed to be properly aligned, and read the
+            # float/double from there
             input = fmtiter.read(size)
-            val = str_storage_getitem(TYPE, input, 0)
+            val = StringBuffer(input).typed_read(TYPE, 0)
         fmtiter.appendobj(float(val))
     return unpack_ieee
 
diff --git a/rpython/rlib/strstorage.py b/rpython/rlib/strstorage.py
deleted file mode 100644
--- a/rpython/rlib/strstorage.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Support for str_storage: i.e., reading primitive types out of RPython string
-#
-# There are various possible ways to implement it, however not all of them are
-# easily supported by the JIT:
-#
-#   1. use _get_raw_str_buf and cast the chars buffer to RAW_STORAGE_PTR: this
-#      works well without the JIT, but the cast to RAW_STORAGE_PTR needs to
-#      happen inside a short "no GC" section (like the one in
-#      rstr.py:copy_string_contents), which has no chance to work during
-#      tracing
-#
-#   2. cast rpy_string to a GcStruct which has the very
-#      same layout, with the only difference that its 'chars' field is no
-#      longer an Array(Char) but e.e. an Array(Signed). Then, we just need to
-#      read the appropriate index into the array.  To support this solution,
-#      the JIT's optimizer needed a few workarounds.  This was removed.
-#
-#   3. use the newly introduced 'llop.gc_load_indexed'.
-#
-
-
-from rpython.rtyper.lltypesystem import lltype, llmemory
-from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.lltypesystem.rstr import STR
-from rpython.rtyper.annlowlevel import llstr
-from rpython.rlib.objectmodel import specialize
-
-
- at specialize.ll()
-def str_storage_getitem(TP, s, byte_offset):
-    # WARNING: the 'byte_offset' is, as its name says, measured in bytes;
-    # however, it should be aligned for TP, otherwise on some platforms this
-    # code will crash!
-    lls = llstr(s)
-    base_ofs = (llmemory.offsetof(STR, 'chars') +
-                llmemory.itemoffsetof(STR.chars, 0))
-    scale_factor = llmemory.sizeof(lltype.Char)
-    return llop.gc_load_indexed(TP, lls, byte_offset,
-                                scale_factor, base_ofs)
diff --git a/rpython/rlib/test/test_strstorage.py b/rpython/rlib/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/rlib/test/test_strstorage.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import py
-import sys
-import struct
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.strstorage import str_storage_getitem
-from rpython.rlib.rarithmetic import r_singlefloat
-from rpython.rtyper.test.tool import BaseRtypingTest
-
-IS_32BIT = (sys.maxint == 2147483647)
-
-class BaseStrStorageTest:
-
-    ## def test_str_getitem_supported(self):
-    ##     if IS_32BIT:
-    ##         expected = False
-    ##     else:
-    ##         expected = True
-    ##     #
-    ##     assert self.str_storage_supported(rffi.LONGLONG) == expected
-    ##     assert self.str_storage_supported(rffi.DOUBLE) == expected
-
-    def test_signed(self):
-        buf = struct.pack('@ll', 42, 43)
-        size = struct.calcsize('@l')
-        assert self.str_storage_getitem(lltype.Signed, buf, 0) == 42
-        assert self.str_storage_getitem(lltype.Signed, buf, size) == 43
-
-    def test_short(self):
-        buf = struct.pack('@hh', 42, 43)
-        size = struct.calcsize('@h')
-        x = self.str_storage_getitem(rffi.SHORT, buf, 0)
-        assert int(x) == 42
-        x = self.str_storage_getitem(rffi.SHORT, buf, size)
-        assert int(x) == 43
-
-    def test_float(self):
-        ## if not str_storage_supported(lltype.Float):
-        ##     py.test.skip('str_storage_getitem(lltype.Float) not supported on this machine')
-        buf = struct.pack('@dd', 12.3, 45.6)
-        size = struct.calcsize('@d')
-        assert self.str_storage_getitem(lltype.Float, buf, 0) == 12.3
-        assert self.str_storage_getitem(lltype.Float, buf, size) == 45.6
-
-    def test_singlefloat(self):
-        buf = struct.pack('@ff', 12.3, 45.6)
-        size = struct.calcsize('@f')
-        x = self.str_storage_getitem(lltype.SingleFloat, buf, 0)
-        assert x == r_singlefloat(12.3)
-        x = self.str_storage_getitem(lltype.SingleFloat, buf, size)
-        assert x == r_singlefloat(45.6)
-
-
-class TestDirect(BaseStrStorageTest):
-
-    ## def str_storage_supported(self, TYPE):
-    ##     return str_storage_supported(TYPE)
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        return str_storage_getitem(TYPE, buf, offset)
-
-class TestRTyping(BaseStrStorageTest, BaseRtypingTest):
-
-    ## def str_storage_supported(self, TYPE):
-    ##     def fn():
-    ##         return str_storage_supported(TYPE)
-    ##     return self.interpret(fn, [])
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        def fn(offset):
-            return str_storage_getitem(TYPE, buf, offset)
-        return self.interpret(fn, [offset])
-
-
-class TestCompiled(BaseStrStorageTest):
-    cache = {}
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        if TYPE not in self.cache:
-            from rpython.translator.c.test.test_genc import compile
-
-            assert isinstance(TYPE, lltype.Primitive)
-            if TYPE in (lltype.Float, lltype.SingleFloat):
-                TARGET_TYPE = lltype.Float
-            else:
-                TARGET_TYPE = lltype.Signed
-
-            def llf(buf, offset):
-                x = str_storage_getitem(TYPE, buf, offset)
-                return lltype.cast_primitive(TARGET_TYPE, x)
-
-            fn = compile(llf, [str, int])
-            self.cache[TYPE] = fn
-        #
-        fn = self.cache[TYPE]
-        x = fn(buf, offset)
-        return lltype.cast_primitive(TYPE, x)

From pypy.commits at gmail.com  Tue May  9 19:29:05 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 16:29:05 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add rlib.Buffer.typed_read,
 and implement it for RawBuffer
Message-ID: <591250c1.e22a370a.3d088.7349@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91217:1bb9b4819612
Date: 2017-05-09 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1bb9b4819612/

Log:	add rlib.Buffer.typed_read, and implement it for RawBuffer

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -18,6 +18,12 @@
     specific type of buffer, or because of alignment issues.
     """
 
+class CannotWrite(Exception):
+    """
+    Raised by Buffer.typed_write in case it is not possible to accomplish the
+    request
+    """
+
 class Buffer(object):
     """Abstract base class for buffers."""
     _attrs_ = ['readonly']
@@ -91,6 +97,13 @@
         """
         raise CannotRead
 
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        """
+        Write the value of type TP at byte_offset. No bounds checks
+        """
+        raise CannotWrite
+
 
 class RawBuffer(Buffer):
     """
@@ -111,6 +124,17 @@
         ptr = self.get_raw_address()
         return llop.raw_load(TP, ptr, byte_offset)
 
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        """
+        Write the value of type TP at byte_offset. No bounds checks
+        """
+        if self.readonly:
+            raise CannotWrite
+        ptr = self.get_raw_address()
+        value = lltype.cast_primitive(TP, value)
+        return llop.raw_store(lltype.Void, ptr, byte_offset, value)
+
 
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -8,14 +8,19 @@
 
 class MyRawBuffer(RawBuffer):
 
-    def __init__(self, data):
-        self._buf = lltype.malloc(rffi.CCHARP.TO, len(data), flavor='raw')
+    def __init__(self, data, readonly):
+        self.readonly = readonly
+        self._n = len(data)
+        self._buf = lltype.malloc(rffi.CCHARP.TO, self._n, flavor='raw')
         for i, ch in enumerate(data):
             self._buf[i] = ch
 
     def get_raw_address(self):
         return self._buf
 
+    def as_str(self):
+        return rffi.charpsize2str(self._buf, self._n)
+
     def __del__(self):
         lltype.free(self._buf, flavor='raw')
         self._buf = None
@@ -119,6 +124,14 @@
         return buf.typed_read(TYPE, offset)
     
 
+class TestRawBufferTypedWrite(object):
+
+    def test_typed_write(self):
+        buf = MyRawBuffer('\xff' * 8, readonly=False)
+        buf.typed_write(rffi.USHORT, 0, 0xABCD)
+        assert buf.as_str() == '\xcd\xab\xff\xff\xff\xff\xff\xff'
+        assert buf.typed_read(rffi.USHORT, 0) == 0xABCD
+    
 
 class TestCompiled(BaseTypedReadTest):
     cache = {}

From pypy.commits at gmail.com  Tue May  9 19:29:08 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 09 May 2017 16:29:08 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add MutableStringBuffer,
 which will be used by struct.pack to incrementally build the
 desired string
Message-ID: <591250c4.4388370a.6c144.6cd6@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91218:ac79316e004f
Date: 2017-05-10 01:28 +0200
http://bitbucket.org/pypy/pypy/changeset/ac79316e004f/

Log:	add MutableStringBuffer, which will be used by struct.pack to
	incrementally build the desired string

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/mutbuffer.py
@@ -0,0 +1,40 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.annlowlevel import llstr, hlstr
+from rpython.rlib.buffer import Buffer
+
+class MutableStringBuffer(Buffer):
+    """
+    A writeable buffer to incrementally fill a string of a fixed size.
+
+    You can fill the string by calling setitem, setslice and typed_write, and
+    get the result by calling finish().
+
+    After you call finish(), you can no longer modify the buffer. There is no
+    check, you will probably get a segfault after translation.
+
+    You can call finish() only once.
+    """
+    _attrs_ = ['readonly', 'll_val']
+    _immutable_ = True
+
+    def __init__(self, size):
+        self.readonly = False
+        # rstr.mallocstr does not pass zero=True, so we call lltype.malloc
+        # directly
+        self.ll_val = lltype.malloc(STR, size, zero=True)
+
+    def finish(self):
+        if not self.ll_val:
+            raise ValueError("Cannot call finish() twice")
+        result = hlstr(self.ll_val)
+        self.ll_val = lltype.nullptr(STR)
+        self.readonly = True
+        return result
+
+    def as_str(self):
+        raise ValueError('as_str() is not supported. Use finish() instead')
+
+    def setitem(self, index, char):
+        self.ll_val.chars[index] = char
diff --git a/rpython/rlib/rstruct/formatiterator.py b/rpython/rlib/rstruct/formatiterator.py
--- a/rpython/rlib/rstruct/formatiterator.py
+++ b/rpython/rlib/rstruct/formatiterator.py
@@ -116,6 +116,7 @@
 def table2desclist(table):
     items = table.items()
     items.sort()
+    import pdb;pdb.set_trace()
     lst = [FmtDesc(key, attrs) for key, attrs in items]
     return unrolling_iterable(lst)
 
diff --git a/rpython/rlib/test/test_mutbuffer.py b/rpython/rlib/test/test_mutbuffer.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/test/test_mutbuffer.py
@@ -0,0 +1,24 @@
+import pytest
+from rpython.rlib.mutbuffer import MutableStringBuffer
+
+class TestMutableStringBuffer(object):
+
+    def test_finish(self):
+        buf = MutableStringBuffer(4)
+        pytest.raises(ValueError, "buf.as_str()")
+        s = buf.finish()
+        assert s == '\x00' * 4
+        pytest.raises(ValueError, "buf.finish()")
+
+    def test_setitem(self):
+        buf = MutableStringBuffer(4)
+        buf.setitem(0, 'A')
+        buf.setitem(1, 'B')
+        buf.setitem(2, 'C')
+        buf.setitem(3, 'D')
+        assert buf.finish() == 'ABCD'
+
+    def test_setslice(self):
+        buf = MutableStringBuffer(6)
+        buf.setslice(2, 'ABCD')
+        assert buf.finish() == '\x00\x00ABCD'

From pypy.commits at gmail.com  Wed May 10 08:25:09 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 10 May 2017 05:25:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: win32: use jom.exe for parallel builds
 if available
Message-ID: <591306a5.9163370a.31f05.c279@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91219:e1fb7a07c627
Date: 2017-05-10 15:24 +0300
http://bitbucket.org/pypy/pypy/changeset/e1fb7a07c627/

Log:	win32: use jom.exe for parallel builds if available

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -119,6 +119,31 @@
     log.error("Could not find a Microsoft Compiler")
     # Assume that the compiler is already part of the environment
 
+# copied from distutils.spawn
+def _find_executable(executable, path=None):
+    """Tries to find 'executable' in the directories listed in 'path'.
+
+    A string listing directories separated by 'os.pathsep'; defaults to
+    os.environ['PATH'].  Returns the complete filename or None if not found.
+    """
+    if path is None:
+        path = os.environ['PATH']
+    paths = path.split(os.pathsep)
+
+    for ext in '.exe', '':
+        newexe = executable + ext
+
+        if os.path.isfile(newexe):
+            return newexe
+        else:
+            for p in paths:
+                f = os.path.join(p, newexe)
+                if os.path.isfile(f):
+                    # the file exists, we have a shot at spawn working
+                    return f
+    return None
+
+
 class MsvcPlatform(Platform):
     name = "msvc"
     so_ext = 'dll'
@@ -128,6 +153,9 @@
 
     cc = 'cl.exe'
     link = 'link.exe'
+    make = 'nmake'
+    if _find_executable('jom.exe'):
+        make = 'jom.exe'
 
     cflags = ('/MD', '/O2', '/Zi')
     link_flags = ('/debug','/LARGEADDRESSAWARE')
@@ -483,11 +511,11 @@
             path = path_to_makefile.makefile_dir
         else:
             path = path_to_makefile
-        log.execute('make %s in %s' % (" ".join(extra_opts), path))
+        log.execute('%s %s in %s' % (self.make, " ".join(extra_opts), path))
         oldcwd = path.chdir()
         try:
             returncode, stdout, stderr = _run_subprocess(
-                'nmake',
+                self.make,
                 ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts,
                 env = self.c_environ)
         finally:

From pypy.commits at gmail.com  Wed May 10 10:37:57 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 10 May 2017 07:37:57 -0700 (PDT)
Subject: [pypy-commit] pypy str-measure: new branch
Message-ID: <591325c5.6903c80a.6f61d.dd7d@mx.google.com>

Author: fijal
Branch: str-measure
Changeset: r91220:13296622c1b1
Date: 2017-05-10 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/13296622c1b1/

Log:	new branch


From pypy.commits at gmail.com  Wed May 10 10:38:00 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 10 May 2017 07:38:00 -0700 (PDT)
Subject: [pypy-commit] pypy str-measure: a branch to start measuring how
 strings are used
Message-ID: <591325c8.f631c80a.a717e.e15c@mx.google.com>

Author: fijal
Branch: str-measure
Changeset: r91221:9923d9d8b6c5
Date: 2017-05-10 16:17 +0200
http://bitbucket.org/pypy/pypy/changeset/9923d9d8b6c5/

Log:	a branch to start measuring how strings are used

diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -153,6 +153,7 @@
         Like bytecode_trace() but doesn't invoke any other events besides the
         trace function.
         """
+        frame._frame_counter += 1
         if (frame.get_w_f_trace() is None or self.is_tracing or
             self.gettrace() is None):
             return
diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py
--- a/pypy/interpreter/pyframe.py
+++ b/pypy/interpreter/pyframe.py
@@ -76,6 +76,9 @@
     valuestackdepth = 0 # number of items on valuestack
     lastblock = None
 
+    # XXX string debugging
+    _frame_counter = 0
+
     # other fields:
     
     # builtin - builtin cache, only if honor__builtins__ is True
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -98,6 +98,7 @@
         'decode_long'               : 'interp_magic.decode_long',
         '_promote'                   : 'interp_magic._promote',
         'stack_almost_full'         : 'interp_magic.stack_almost_full',
+        'set_str_debug_file'        : 'interp_debug.set_str_debug_file',
     }
     if sys.platform == 'win32':
         interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py
--- a/pypy/module/__pypy__/interp_debug.py
+++ b/pypy/module/__pypy__/interp_debug.py
@@ -28,3 +28,15 @@
 @jit.dont_look_inside
 def debug_flush(space):
     debug.debug_flush()
+
+class Cache(object):
+    def __init__(self, space):
+        self.w_debug_file = None
+
+def set_str_debug_file(space, w_debug_file):
+    if space.is_none(w_debug_file):
+        w_debug_file = None
+    space.fromcache(Cache).w_debug_file = w_debug_file
+
+def get_str_debug_file(space):
+    return space.fromcache(Cache).w_debug_file
\ No newline at end of file
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -50,15 +50,15 @@
     def nonmovable_carray(self, space):
         return BytearrayBuffer(self.data, False).get_raw_address()
 
-    def _new(self, value):
+    def _new(self, space, value):
         if value is self.data:
             value = value[:]
         return W_BytearrayObject(value)
 
-    def _new_from_buffer(self, buffer):
+    def _new_from_buffer(self, space, buffer):
         return W_BytearrayObject([buffer[i] for i in range(len(buffer))])
 
-    def _new_from_list(self, value):
+    def _new_from_list(self, space, value):
         return W_BytearrayObject(value)
 
     def _empty(self):
@@ -443,10 +443,10 @@
 
     def descr_add(self, space, w_other):
         if isinstance(w_other, W_BytearrayObject):
-            return self._new(self.data + w_other.data)
+            return self._new(space, self.data + w_other.data)
 
         if isinstance(w_other, W_BytesObject):
-            return self._add(self._op_val(space, w_other))
+            return self._add(space, self._op_val(space, w_other))
 
         try:
             buffer = _get_buffer(space, w_other)
@@ -454,11 +454,11 @@
             if e.match(space, space.w_TypeError):
                 return space.w_NotImplemented
             raise
-        return self._add(buffer)
+        return self._add(space, buffer)
 
-    @specialize.argtype(1)
-    def _add(self, other):
-        return self._new(self.data + [other[i] for i in range(len(other))])
+    @specialize.argtype(2)
+    def _add(self, space, other):
+        return self._new(space, self.data + [other[i] for i in range(len(other))])
 
     def descr_reverse(self, space):
         self.data.reverse()
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -481,10 +481,10 @@
                         "found", len(self._value))
         return space.newint(ord(self._value[0]))
 
-    def _new(self, value):
+    def _new(self, space, value):
         return W_BytesObject(value)
 
-    def _new_from_list(self, value):
+    def _new_from_list(self, space, value):
         return W_BytesObject(''.join(value))
 
     def _empty(self):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -361,7 +361,7 @@
     def newunicode(self, uni):
         assert uni is not None
         assert isinstance(uni, unicode)
-        return W_UnicodeObject(uni)
+        return W_UnicodeObject(self, uni)
 
     def type(self, w_obj):
         jit.promote(w_obj.__class__)
diff --git a/pypy/objspace/std/stringmethods.py b/pypy/objspace/std/stringmethods.py
--- a/pypy/objspace/std/stringmethods.py
+++ b/pypy/objspace/std/stringmethods.py
@@ -20,7 +20,7 @@
         #if start == 0 and stop == len(s) and space.is_w(space.type(orig_obj),
         #                                                space.w_bytes):
         #    return orig_obj
-        return self._new(s[start:stop])
+        return self._new(space, s[start:stop])
 
     def _convert_idx_params(self, space, w_start, w_end):
         value = self._val(space)
@@ -61,7 +61,7 @@
                 if e.match(space, space.w_TypeError):
                     return space.w_NotImplemented
                 raise
-            return self._new(self._val(space) + other)
+            return self._new(space, self._val(space) + other)
 
         # Bytearray overrides this method, CPython doesn't support contacting
         # buffers and strs, and unicodes are always handled above
@@ -77,8 +77,8 @@
         if times <= 0:
             return self._empty()
         if self._len() == 1:
-            return self._new(self._multi_chr(self._val(space)[0]) * times)
-        return self._new(self._val(space) * times)
+            return self._new(space, self._multi_chr(self._val(space)[0]) * times)
+        return self._new(space, self._val(space) * times)
 
     descr_rmul = descr_mul
 
@@ -94,7 +94,7 @@
                 return self._sliced(space, selfvalue, start, stop, self)
             else:
                 ret = _descr_getslice_slowpath(selfvalue, start, step, sl)
-                return self._new_from_list(ret)
+                return self._new_from_list(space, ret)
 
         index = space.getindex_w(w_index, space.w_IndexError, "string index")
         return self._getitem_result(space, index)
@@ -105,7 +105,7 @@
             character = selfvalue[index]
         except IndexError:
             raise oefmt(space.w_IndexError, "string index out of range")
-        return self._new(character)
+        return self._new(space, character)
 
     def descr_getslice(self, space, w_start, w_stop):
         selfvalue = self._val(space)
@@ -125,7 +125,7 @@
         builder.append(self._upper(value[0]))
         for i in range(1, len(value)):
             builder.append(self._lower(value[i]))
-        return self._new(builder.build())
+        return self._new(space, builder.build())
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_center(self, space, width, w_fillchar):
@@ -143,7 +143,7 @@
         else:
             centered = value
 
-        return self._new(centered)
+        return self._new(space, centered)
 
     def descr_count(self, space, w_sub, w_start=None, w_end=None):
         value, start, end = self._convert_idx_params(space, w_start, w_end)
@@ -207,7 +207,7 @@
                                                          tabsize) + token
             oldtoken = token
 
-        return self._new(expanded)
+        return self._new(space, expanded)
 
     def _tabindent(self, token, tabsize):
         """calculates distance behind the token to the next tabstop"""
@@ -442,7 +442,7 @@
             if value and i != 0:
                 sb.append(value)
             sb.append(unwrapped[i])
-        return self._new(sb.build())
+        return self._new(space, sb.build())
 
     def _join_autoconvert(self, space, list_w):
         assert False, 'unreachable'
@@ -459,7 +459,7 @@
             fillchar = self._multi_chr(fillchar[0])
             value = value + fillchar * d
 
-        return self._new(value)
+        return self._new(space, value)
 
     @unwrap_spec(width=int, w_fillchar=WrappedDefault(' '))
     def descr_rjust(self, space, width, w_fillchar):
@@ -473,14 +473,14 @@
             fillchar = self._multi_chr(fillchar[0])
             value = d * fillchar + value
 
-        return self._new(value)
+        return self._new(space, value)
 
     def descr_lower(self, space):
         value = self._val(space)
         builder = self._builder(len(value))
         for i in range(len(value)):
             builder.append(self._lower(value[i]))
-        return self._new(builder.build())
+        return self._new(space, builder.build())
 
     def descr_partition(self, space, w_sub):
         from pypy.objspace.std.bytearrayobject import W_BytearrayObject
@@ -501,11 +501,11 @@
 
             pos = find(value, sub, 0, len(value))
             if pos != -1 and isinstance(self, W_BytearrayObject):
-                w_sub = self._new_from_buffer(sub)
+                w_sub = self._new_from_buffer(space, sub)
 
         if pos == -1:
             if isinstance(self, W_BytearrayObject):
-                self = self._new(value)
+                self = self._new(space, value)
             return space.newtuple([self, self._empty(), self._empty()])
         else:
             return space.newtuple(
@@ -531,11 +531,11 @@
 
             pos = rfind(value, sub, 0, len(value))
             if pos != -1 and isinstance(self, W_BytearrayObject):
-                w_sub = self._new_from_buffer(sub)
+                w_sub = self._new_from_buffer(space, sub)
 
         if pos == -1:
             if isinstance(self, W_BytearrayObject):
-                self = self._new(value)
+                self = self._new(space, value)
             return space.newtuple([self._empty(), self._empty(), self])
         else:
             return space.newtuple(
@@ -557,7 +557,7 @@
         except OverflowError:
             raise oefmt(space.w_OverflowError, "replace string is too long")
 
-        return self._new(res)
+        return self._new(space, res)
 
     @unwrap_spec(maxsplit=int)
     def descr_split(self, space, w_sep=None, maxsplit=-1):
@@ -716,13 +716,13 @@
                 builder.append(self._upper(ch))
             else:
                 builder.append(ch)
-        return self._new(builder.build())
+        return self._new(space, builder.build())
 
     def descr_title(self, space):
         selfval = self._val(space)
         if len(selfval) == 0:
             return self
-        return self._new(self.title(selfval))
+        return self._new(space, self.title(selfval))
 
     @jit.elidable
     def title(self, value):
@@ -764,24 +764,24 @@
             for char in string:
                 if not deletion_table[ord(char)]:
                     buf.append(table[ord(char)])
-        return self._new(buf.build())
+        return self._new(space, buf.build())
 
     def descr_upper(self, space):
         value = self._val(space)
         builder = self._builder(len(value))
         for i in range(len(value)):
             builder.append(self._upper(value[i]))
-        return self._new(builder.build())
+        return self._new(space, builder.build())
 
     @unwrap_spec(width=int)
     def descr_zfill(self, space, width):
         selfval = self._val(space)
         if len(selfval) == 0:
-            return self._new(self._multi_chr(self._chr('0')) * width)
+            return self._new(space, self._multi_chr(self._chr('0')) * width)
         num_zeros = width - len(selfval)
         if num_zeros <= 0:
             # cannot return self, in case it is a subclass of str
-            return self._new(selfval)
+            return self._new(space, selfval)
 
         builder = self._builder(width)
         if len(selfval) > 0 and (selfval[0] == '+' or selfval[0] == '-'):
@@ -792,10 +792,10 @@
             start = 0
         builder.append_multiple_char(self._chr('0'), num_zeros)
         builder.append_slice(selfval, start, len(selfval))
-        return self._new(builder.build())
+        return self._new(space, builder.build())
 
     def descr_getnewargs(self, space):
-        return space.newtuple([self._new(self._val(space))])
+        return space.newtuple([self._new(space, self._val(space))])
 
 # ____________________________________________________________
 # helpers for slow paths, moved out because they contain loops
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1,5 +1,6 @@
 """The builtin unicode implementation"""
 
+import py
 from rpython.rlib.objectmodel import (
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs)
@@ -29,11 +30,26 @@
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
+    _frame_counter = 0
+    _frame_id = 0
 
     @enforceargs(uni=unicode)
-    def __init__(self, unistr):
+    def __init__(self, space, unistr):
         assert isinstance(unistr, unicode)
         self._value = unistr
+        if space is None:
+            return
+        frame = space.getexecutioncontext().gettopframe()
+        if frame is None:
+            return
+        self._frame_counter = frame._frame_counter
+        self._frame_id = compute_unique_id(frame)
+        from pypy.module.__pypy__.interp_debug import get_str_debug_file
+        w_file = get_str_debug_file(space)
+        if w_file is None:
+            return
+        space.call_function(space.getattr(w_file, space.newtext("write")),
+            space.newtext("descr_new %s %s\n" % (self._frame_counter, self._frame_id)))
 
     def __repr__(self):
         """representation for debugging purposes"""
@@ -43,10 +59,10 @@
         # for testing
         return self._value
 
-    def create_if_subclassed(self):
+    def create_if_subclassed(self, space):
         if type(self) is W_UnicodeObject:
             return self
-        return W_UnicodeObject(self._value)
+        return W_UnicodeObject(space, self._value)
 
     def is_w(self, space, w_other):
         if not isinstance(w_other, W_UnicodeObject):
@@ -105,11 +121,11 @@
                          "found", len(self._value))
         return space.newint(ord(self._value[0]))
 
-    def _new(self, value):
-        return W_UnicodeObject(value)
+    def _new(self, space, value):
+        return W_UnicodeObject(space, value)
 
-    def _new_from_list(self, value):
-        return W_UnicodeObject(u''.join(value))
+    def _new_from_list(self, space, value):
+        return W_UnicodeObject(space, u''.join(value))
 
     def _empty(self):
         return W_UnicodeObject.EMPTY
@@ -222,7 +238,7 @@
 
         assert isinstance(w_value, W_UnicodeObject)
         w_newobj = space.allocate_instance(W_UnicodeObject, w_unicodetype)
-        W_UnicodeObject.__init__(w_newobj, w_value._value)
+        W_UnicodeObject.__init__(w_newobj, space, w_value._value)
         return w_newobj
 
     def descr_repr(self, space):
@@ -354,7 +370,7 @@
                     raise oefmt(space.w_TypeError,
                                 "character mapping must return integer, None "
                                 "or unicode")
-        return W_UnicodeObject(u''.join(result))
+        return W_UnicodeObject(space, u''.join(result))
 
     def descr_encode(self, space, w_encoding=None, w_errors=None):
         encoding, errors = _get_encoding_and_errors(space, w_encoding,
@@ -420,7 +436,7 @@
 
 
 def wrapunicode(space, uni):
-    return W_UnicodeObject(uni)
+    return W_UnicodeObject(space, uni)
 
 
 def plain_str2unicode(space, s):
@@ -562,7 +578,7 @@
         return unicode_from_encoded_object(space, w_bytes, encoding, "strict")
     s = space.bytes_w(w_bytes)
     try:
-        return W_UnicodeObject(s.decode("ascii"))
+        return W_UnicodeObject(space, s.decode("ascii"))
     except UnicodeDecodeError:
         # raising UnicodeDecodeError is messy, "please crash for me"
         return unicode_from_encoded_object(space, w_bytes, "ascii", "strict")
@@ -967,6 +983,39 @@
         of the specified width. The string S is never truncated.
         """
 
+def setup():
+    from pypy.module.__pypy__.interp_debug import get_str_debug_file
+
+    def wrap(func):
+        d = {'orig': func, 'get_str_debug_file': get_str_debug_file}
+        name = func.__name__
+        orig_args = list(func.__code__.co_varnames[:func.__code__.co_argcount])
+        args = orig_args[:]
+        if func.func_defaults:
+            i = func.__code__.co_argcount - len(func.func_defaults)
+            for j, default in enumerate(func.func_defaults):
+                args[i] = "%s = %r" % (args[i], func.func_defaults[j])
+                i += 1
+        func_args = ", ".join(args)
+        lines = ["def %s(%s):" % (name, func_args),
+        "    w_file = get_str_debug_file(space)",
+        "    if w_file is not None:",
+        "        txt = '%s ' + str(self._frame_counter) + ' ' + str(self._frame_id) + ' '+ '\\n'" % func.func_name,
+        "        space.call_function(space.getattr(w_file, space.newtext('write')), space.newtext(txt))",
+        "    return orig(%s)" % (", ".join(orig_args),)]
+        exec "\n".join(lines) in d
+        if hasattr(func, 'unwrap_spec'):
+            d[name].unwrap_spec = func.unwrap_spec
+        # get it as an unbound method
+        return d[name]
+
+    for k, v in W_UnicodeObject.__dict__.iteritems():
+        if k == 'descr_new':
+            continue
+        if k.startswith('descr_'):
+           setattr(W_UnicodeObject, k, wrap(getattr(W_UnicodeObject, k)))
+
+setup()
 
 W_UnicodeObject.typedef = TypeDef(
     "unicode", basestring_typedef,
@@ -1112,7 +1161,7 @@
     return [s for s in value]
 
 
-W_UnicodeObject.EMPTY = W_UnicodeObject(u'')
+W_UnicodeObject.EMPTY = W_UnicodeObject(None, u'')
 
 
 # Helper for converting int/long
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -16,6 +16,7 @@
 
     standalone_only = ('-mdynamic-no-pic',)
     shared_only = ()
+    accepts_flto = False
 
     link_flags = (DARWIN_VERSION_MIN,)
     cflags = ('-O3', '-fomit-frame-pointer', DARWIN_VERSION_MIN)
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -11,6 +11,7 @@
     exe_ext = ''
     make_cmd = 'make'
     so_prefixes = ('lib', '')
+    accepts_flto = True
 
     relevant_environ = ('CPATH', 'LIBRARY_PATH', 'C_INCLUDE_PATH')
 
@@ -131,7 +132,7 @@
             cflags = tuple(self.cflags) + tuple(self.standalone_only)
 
         # xxx check which compilers accept this option or not
-        if not config or config.translation.gcrootfinder != 'asmgcc':
+        if (not config or config.translation.gcrootfinder != 'asmgcc') and self.accepts_flto:
             cflags = ('-flto',) + cflags
 
         m = GnuMakefile(path)

From pypy.commits at gmail.com  Wed May 10 13:39:52 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 10 May 2017 10:39:52 -0700 (PDT)
Subject: [pypy-commit] pypy str-measure: use a different id
Message-ID: <59135068.458e370a.d538d.02e1@mx.google.com>

Author: fijal
Branch: str-measure
Changeset: r91222:3b8e8de4a414
Date: 2017-05-10 19:37 +0200
http://bitbucket.org/pypy/pypy/changeset/3b8e8de4a414/

Log:	use a different id

diff --git a/pypy/module/__pypy__/interp_debug.py b/pypy/module/__pypy__/interp_debug.py
--- a/pypy/module/__pypy__/interp_debug.py
+++ b/pypy/module/__pypy__/interp_debug.py
@@ -32,6 +32,7 @@
 class Cache(object):
     def __init__(self, space):
         self.w_debug_file = None
+        self.counter = 0
 
 def set_str_debug_file(space, w_debug_file):
     if space.is_none(w_debug_file):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -30,8 +30,7 @@
 class W_UnicodeObject(W_Root):
     import_from_mixin(StringMethods)
     _immutable_fields_ = ['_value']
-    _frame_counter = 0
-    _frame_id = 0
+    _string_id = 0
 
     @enforceargs(uni=unicode)
     def __init__(self, space, unistr):
@@ -42,14 +41,15 @@
         frame = space.getexecutioncontext().gettopframe()
         if frame is None:
             return
-        self._frame_counter = frame._frame_counter
-        self._frame_id = compute_unique_id(frame)
-        from pypy.module.__pypy__.interp_debug import get_str_debug_file
+        from pypy.module.__pypy__.interp_debug import get_str_debug_file, Cache
+        cache = space.fromcache(Cache)
+        self._string_id = cache.counter
+        cache.counter += 1
         w_file = get_str_debug_file(space)
         if w_file is None:
             return
         space.call_function(space.getattr(w_file, space.newtext("write")),
-            space.newtext("descr_new %s %s\n" % (self._frame_counter, self._frame_id)))
+            space.newtext("%d = descr_new\n" % (self._string_id,)))
 
     def __repr__(self):
         """representation for debugging purposes"""
@@ -1000,7 +1000,7 @@
         lines = ["def %s(%s):" % (name, func_args),
         "    w_file = get_str_debug_file(space)",
         "    if w_file is not None:",
-        "        txt = '%s ' + str(self._frame_counter) + ' ' + str(self._frame_id) + ' '+ '\\n'" % func.func_name,
+        "        txt = '%s ' + str(self._string_id) + '\\n'" % func.func_name,
         "        space.call_function(space.getattr(w_file, space.newtext('write')), space.newtext(txt))",
         "    return orig(%s)" % (", ".join(orig_args),)]
         exec "\n".join(lines) in d

From pypy.commits at gmail.com  Wed May 10 14:04:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 10 May 2017 11:04:10 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Update micronumpy
Message-ID: <5913561a.8f35ed0a.7a0ca.0ca6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91223:4c49b02f7487
Date: 2017-05-10 19:03 +0100
http://bitbucket.org/pypy/pypy/changeset/4c49b02f7487/

Log:	Update micronumpy

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
@@ -21,7 +22,7 @@
 
 TimSort = make_timsort_class()
 class StrideSort(TimSort):
-    ''' 
+    '''
     argsort (return the indices to sort) a list of strides
     '''
     def __init__(self, rangelist, strides, order):
@@ -380,14 +381,14 @@
 
     def get_buffer(self, space, flags):
         errtype = space.w_ValueError # should be BufferError, numpy does this instead
-        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_C_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not C-contiguous")
-        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
@@ -397,7 +398,7 @@
             not self.flags & NPY.ARRAY_WRITEABLE):
            raise oefmt(errtype, "buffer source array is read-only")
         readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
-        return ArrayBuffer(self, readonly)
+        return ArrayView(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
         # copy the general pattern of the strides
@@ -527,7 +528,7 @@
         try:
             length = support.product_check(shape)
             self.size = ovfcheck(length * dtype.elsize)
-        except OverflowError: 
+        except OverflowError:
             raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
         if storage == lltype.nullptr(RAW_STORAGE):
             if dtype.num == NPY.OBJECT:
@@ -701,10 +702,8 @@
     def __del__(self):
         free_raw_storage(self.storage)
 
-
-class ArrayBuffer(Buffer):
+class ArrayData(Buffer):
     _immutable_ = True
-
     def __init__(self, impl, readonly):
         self.impl = impl
         self.readonly = readonly
@@ -725,6 +724,28 @@
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
 
+
+class ArrayView(BufferView):
+    _immutable_ = True
+
+    def __init__(self, impl, readonly):
+        self.impl = impl
+        self.readonly = readonly
+        self.data = ArrayData(impl, readonly)
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def as_readbuf(self):
+        return ArrayData(self.impl, readonly=True)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return ArrayData(self.impl, readonly=False)
+
     def getformat(self):
         sb = StringBuilder()
         self.impl.dtype.getformat(sb)
@@ -741,5 +762,3 @@
 
     def getstrides(self):
         return self.impl.strides
-
-
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -3,6 +3,7 @@
     WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, \
     make_weakref_descr
+from pypy.interpreter.buffer import SimpleView
 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import RAW_STORAGE_PTR
@@ -806,17 +807,9 @@
     def buffer_w(self, space, flags):
         return self.implementation.get_buffer(space, flags)
 
-    def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
-
-    def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL)
-
-    def charbuf_w(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
-
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
+        return space.newbuffer(
+            self.implementation.get_buffer(space, space.BUF_FULL).as_writebuf())
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):

From pypy.commits at gmail.com  Wed May 10 14:15:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 10 May 2017 11:15:58 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Add missing
 W_MemoryView.copy()
Message-ID: <591358de.7633c80a.ef715.0f59@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91224:ed953d9e05ef
Date: 2017-05-10 19:15 +0100
http://bitbucket.org/pypy/pypy/changeset/ed953d9e05ef/

Log:	Add missing W_MemoryView.copy()

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -128,6 +128,12 @@
         else:
             raise TypeError("memoryview: invalid slice key")
 
+    @staticmethod
+    def copy(w_view):
+        # TODO suboffsets
+        view = w_view.view
+        return W_MemoryView(view)
+
     def descr_setitem(self, space, w_index, w_obj):
         self._check_released(space)
         if self.view.readonly:

From pypy.commits at gmail.com  Wed May 10 19:05:25 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:25 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add more tests
Message-ID: <59139cb5.0e3eed0a.e93a.1ef7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91226:6a63a61cf847
Date: 2017-05-10 11:32 +0200
http://bitbucket.org/pypy/pypy/changeset/6a63a61cf847/

Log:	add more tests

diff --git a/rpython/rlib/rstruct/formatiterator.py b/rpython/rlib/rstruct/formatiterator.py
--- a/rpython/rlib/rstruct/formatiterator.py
+++ b/rpython/rlib/rstruct/formatiterator.py
@@ -116,7 +116,6 @@
 def table2desclist(table):
     items = table.items()
     items.sort()
-    import pdb;pdb.set_trace()
     lst = [FmtDesc(key, attrs) for key, attrs in items]
     return unrolling_iterable(lst)
 
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -60,6 +60,30 @@
         self.check('f', 123.456)
         self.check('d', 123.456789)
 
+    def test_pack_char(self):
+        self.check('c', 'a')
+
+    def test_pack_pad(self):
+        bigendian = self.endianess == '>'
+        fmtiter = FakeFormatIter(bigendian, None)
+        standardfmttable.pack_pad(fmtiter, 4)
+        s = fmtiter.result.build()
+        assert s == '\x00'*4
+
+    def test_pack_string(self):
+        bigendian = self.endianess == '>'
+        fmtiter = FakeFormatIter(bigendian, 'hello')
+        standardfmttable.pack_string(fmtiter, 8)
+        s = fmtiter.result.build()
+        assert s == 'hello\x00\x00\x00'
+
+    def test_pack_pascal(self):
+        bigendian = self.endianess == '>'
+        fmtiter = FakeFormatIter(bigendian, 'hello')
+        standardfmttable.pack_pascal(fmtiter, 8)
+        s = fmtiter.result.build()
+        assert s == '\x05hello\x00\x00'
+
 
 class TestPackLittleEndian(BaseTestPack):
     endianess = '<'

From pypy.commits at gmail.com  Wed May 10 19:05:27 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:27 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: break the world,
 and start to refactor pack_* to build the result inside a
 MutableStringBuffer instead of a StringBuilder
Message-ID: <59139cb7.8801370a.18860.1c6d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91227:b6e312499e57
Date: 2017-05-10 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/b6e312499e57/

Log:	WIP: break the world, and start to refactor pack_* to build the
	result inside a MutableStringBuffer instead of a StringBuilder

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -23,8 +23,12 @@
         self.readonly = False
         # rstr.mallocstr does not pass zero=True, so we call lltype.malloc
         # directly
+        self.size = size
         self.ll_val = lltype.malloc(STR, size, zero=True)
 
+    def getlength(self):
+        return self.size
+
     def finish(self):
         if not self.ll_val:
             raise ValueError("Cannot call finish() twice")
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -13,7 +13,6 @@
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.buffer import StringBuffer
-#from rpython.rlib.strstorage import str_storage_getitem
 from rpython.rlib import rarithmetic
 from rpython.rlib.buffer import CannotRead
 from rpython.rtyper.lltypesystem import rffi
@@ -29,7 +28,8 @@
     if len(string) != 1:
         raise StructError("expected a string of length 1")
     c = string[0]   # string->char conversion for the annotator
-    fmtiter.result.append(c)
+    fmtiter.result.setitem(fmtiter.pos, c)
+    fmtiter.advance(1)
 
 def pack_bool(fmtiter):
     c = '\x01' if fmtiter.accept_bool_arg() else '\x00'
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -5,11 +5,15 @@
 
 class FakeFormatIter(object):
 
-    def __init__(self, bigendian, value):
+    def __init__(self, bigendian, size, value):
         from rpython.rlib.rstring import StringBuilder
         self.value = value
         self.bigendian = bigendian
-        self.result = StringBuilder(8)
+        self.result = MutableStringBuffer(size)
+        self.pos = 0
+
+    def advance(self, count):
+        self.pos += count
 
     def _accept_arg(self):
         return self.value
@@ -32,11 +36,12 @@
 
     def mypack(self, fmt, value):
         bigendian = self.endianess == '>'
-        fake_fmtiter = FakeFormatIter(bigendian, value)
+        size = struct.calcsize(fmt)
+        fake_fmtiter = FakeFormatIter(bigendian, size, value)
         attrs = self.fmttable[fmt]
         pack = attrs['pack']
         pack(fake_fmtiter)
-        return fake_fmtiter.result.build()
+        return fake_fmtiter.result.finish()
 
     def check(self, fmt, value):
         expected = struct.pack(self.endianess+fmt, value)

From pypy.commits at gmail.com  Wed May 10 19:05:22 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:22 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add direct tests for the
 various pack() functions in rstruct
Message-ID: <59139cb2.8f35ed0a.bc2fc.21d7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91225:a99c31346a82
Date: 2017-05-10 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/a99c31346a82/

Log:	add direct tests for the various pack() functions in rstruct

diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -0,0 +1,69 @@
+import pytest
+from rpython.rlib.rstruct import standardfmttable
+from rpython.rlib.mutbuffer import MutableStringBuffer
+import struct
+
+class FakeFormatIter(object):
+
+    def __init__(self, bigendian, value):
+        from rpython.rlib.rstring import StringBuilder
+        self.value = value
+        self.bigendian = bigendian
+        self.result = StringBuilder(8)
+
+    def _accept_arg(self):
+        return self.value
+
+    def __getattr__(self, name):
+        if name.startswith('accept_'):
+            return self._accept_arg
+        raise AttributeError(name)
+
+
+class BaseTestPack(object):
+    """
+    These test tests only the various pack_* functions, individually.  There
+    is no RPython interface to them, as for now they are used only to
+    implement struct.pack in pypy/module/struct
+    """
+
+    endianess = None
+    fmttable = standardfmttable.standard_fmttable
+
+    def mypack(self, fmt, value):
+        bigendian = self.endianess == '>'
+        fake_fmtiter = FakeFormatIter(bigendian, value)
+        attrs = self.fmttable[fmt]
+        pack = attrs['pack']
+        pack(fake_fmtiter)
+        return fake_fmtiter.result.build()
+
+    def check(self, fmt, value):
+        expected = struct.pack(self.endianess+fmt, value)
+        got = self.mypack(fmt, value)
+        assert got == expected
+
+    def test_pack_int(self):
+        self.check('b', 42)
+        self.check('B', 242)
+        self.check('h', 32767)
+        self.check('H', 32768)
+        self.check("i", 0x41424344)
+        self.check("i", -3)
+        self.check("i", -2147483648)
+        self.check("I", 0x81424344)
+        self.check("q", 0x4142434445464748)
+        self.check("q", -0x41B2B3B4B5B6B7B8)
+        self.check("Q", 0x8142434445464748)
+
+    def test_pack_ieee(self):
+        self.check('f', 123.456)
+        self.check('d', 123.456789)
+
+
+class TestPackLittleEndian(BaseTestPack):
+    endianess = '<'
+
+
+class TestPackBigEndian(BaseTestPack):
+    endianess = '>'

From pypy.commits at gmail.com  Wed May 10 19:05:30 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:30 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix pack_int for the new
 interface
Message-ID: <59139cba.d465370a.a83d9.1e98@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91228:addb00e213ad
Date: 2017-05-10 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/addb00e213ad/

Log:	fix pack_int for the new interface

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -119,14 +119,18 @@
         value = method()
         if not min <= value <= max:
             raise StructError(errormsg)
+        #
+        pos = fmtiter.pos + size - 1        
         if fmtiter.bigendian:
             for i in unroll_revrange_size:
                 x = (value >> (8*i)) & 0xff
-                fmtiter.result.append(chr(x))
+                fmtiter.result.setitem(pos-i, chr(x))
         else:
+
             for i in unroll_revrange_size:
-                fmtiter.result.append(chr(value & 0xff))
+                fmtiter.result.setitem(pos-i, chr(value & 0xff))
                 value >>= 8
+        fmtiter.advance(size)
 
     _memo[key] = pack_int
     return pack_int

From pypy.commits at gmail.com  Wed May 10 19:05:35 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:35 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix pack_bool and pack_string
Message-ID: <59139cbf.2622ed0a.51028.1dcb@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91230:bec494d81ed2
Date: 2017-05-10 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/bec494d81ed2/

Log:	fix pack_bool and pack_string

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -33,15 +33,20 @@
 
 def pack_bool(fmtiter):
     c = '\x01' if fmtiter.accept_bool_arg() else '\x00'
-    fmtiter.result.append(c)
+    fmtiter.result.setitem(fmtiter.pos, c)
+    fmtiter.advance(1)
 
 def pack_string(fmtiter, count):
+    pos = fmtiter.pos
     string = fmtiter.accept_str_arg()
     if len(string) < count:
-        fmtiter.result.append(string)
-        fmtiter.result.append_multiple_char('\x00', count - len(string))
+        fmtiter.result.setslice(pos, string)
+        if fmtiter.needs_zeros:
+            for i in range(pos + len(string), count):
+                fmtiter.result.setitem(i, '\x00')
     else:
-        fmtiter.result.append_slice(string, 0, count)
+        fmtiter.result.setslice(pos, string[:count])
+    fmtiter.advance(count)
 
 def pack_pascal(fmtiter, count):
     string = fmtiter.accept_str_arg()
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -10,7 +10,11 @@
         self.value = value
         self.bigendian = bigendian
         self.result = MutableStringBuffer(size)
+        # we set the buffer to non-zero, so ensure that we actively write 0s
+        # where it's needed
+        self.result.setslice(0, '\xff'*size)
         self.pos = 0
+        self.needs_zeros = True
 
     def advance(self, count):
         self.pos += count
@@ -48,6 +52,12 @@
         pack(fake_fmtiter)
         return fake_fmtiter.finish()
 
+    def mypack_fn(self, func, size, arg, value):
+        bigendian = self.endianess == '>'
+        fmtiter = FakeFormatIter(bigendian, size, value)
+        func(fmtiter, arg)
+        return fmtiter.finish()
+
     def check(self, fmt, value):
         expected = struct.pack(self.endianess+fmt, value)
         got = self.mypack(fmt, value)
@@ -73,6 +83,10 @@
     def test_pack_char(self):
         self.check('c', 'a')
 
+    def test_pack_bool(self):
+        self.check('?', True)
+        self.check('?', False)
+
     def test_pack_pad(self):
         bigendian = self.endianess == '>'
         fmtiter = FakeFormatIter(bigendian, None)
@@ -81,11 +95,13 @@
         assert s == '\x00'*4
 
     def test_pack_string(self):
-        bigendian = self.endianess == '>'
-        fmtiter = FakeFormatIter(bigendian, 'hello')
-        standardfmttable.pack_string(fmtiter, 8)
-        s = fmtiter.result.build()
+        s = self.mypack_fn(standardfmttable.pack_string,
+                           arg=8, value='hello', size=8)
         assert s == 'hello\x00\x00\x00'
+        #
+        s = self.mypack_fn(standardfmttable.pack_string,
+                           arg=8, value='hello world', size=8)
+        assert s == 'hello wo'
 
     def test_pack_pascal(self):
         bigendian = self.endianess == '>'

From pypy.commits at gmail.com  Wed May 10 19:05:38 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:38 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix pack_pad
Message-ID: <59139cc2.d1efe90a.82eee.1d56@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91231:574964807a61
Date: 2017-05-10 18:24 +0200
http://bitbucket.org/pypy/pypy/changeset/574964807a61/

Log:	fix pack_pad

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -21,7 +21,11 @@
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
 
 def pack_pad(fmtiter, count):
-    fmtiter.result.append_multiple_char('\x00', count)
+    if fmtiter.needs_zeros:
+        pos = fmtiter.pos
+        for i in range(count):
+            fmtiter.result.setitem(pos+i, '\x00')
+    fmtiter.advance(count)
 
 def pack_char(fmtiter):
     string = fmtiter.accept_str_arg()
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -88,10 +88,8 @@
         self.check('?', False)
 
     def test_pack_pad(self):
-        bigendian = self.endianess == '>'
-        fmtiter = FakeFormatIter(bigendian, None)
-        standardfmttable.pack_pad(fmtiter, 4)
-        s = fmtiter.result.build()
+        s = self.mypack_fn(standardfmttable.pack_pad,
+                           arg=4, value=None, size=4)
         assert s == '\x00'*4
 
     def test_pack_string(self):

From pypy.commits at gmail.com  Wed May 10 19:05:33 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:33 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: migrate pack_float to the new
 buffer-based interface
Message-ID: <59139cbd.2823ed0a.d1d50.1da4@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91229:f3fc7a2efd43
Date: 2017-05-10 17:44 +0200
http://bitbucket.org/pypy/pypy/changeset/f3fc7a2efd43/

Log:	migrate pack_float to the new buffer-based interface

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -16,7 +16,7 @@
 
     You can call finish() only once.
     """
-    _attrs_ = ['readonly', 'll_val']
+    _attrs_ = ['readonly', 'll_val', 'size']
     _immutable_ = True
 
     def __init__(self, size):
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -265,14 +265,17 @@
     return (mant, (sign << BITS - MANT_DIG - 1) | exp)
 
 @jit.unroll_safe
-def pack_float(result, x, size, be):
-    l = []
+def pack_float(result, pos, x, size, be):
     unsigned = float_pack(x, size)
-    for i in range(size):
-        l.append(chr((unsigned >> (i * 8)) & 0xFF))
     if be:
-        l.reverse()
-    result.append("".join(l))
+        # write in reversed order
+        for i in range(size):
+            c = chr((unsigned >> (i * 8)) & 0xFF)
+            result.setitem(pos + size - i - 1, c)
+    else:
+        for i in range(size):
+            c = chr((unsigned >> (i * 8)) & 0xFF)
+            result.setitem(pos+i, c)
 
 @jit.unroll_safe
 def pack_float80(result, x, size, be):
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -62,10 +62,14 @@
     def packer(fmtiter):
         fl = fmtiter.accept_float_arg()
         try:
-            return ieee.pack_float(fmtiter.result, fl, size, fmtiter.bigendian)
+            result = ieee.pack_float(fmtiter.result, fmtiter.pos,
+                                     fl, size, fmtiter.bigendian)
         except OverflowError:
             assert size == 4
             raise StructOverflowError("float too large for format 'f'")
+        else:
+            fmtiter.advance(size)
+            return result
     return packer
 
 # ____________________________________________________________
diff --git a/rpython/rlib/rstruct/test/test_ieee.py b/rpython/rlib/rstruct/test/test_ieee.py
--- a/rpython/rlib/rstruct/test/test_ieee.py
+++ b/rpython/rlib/rstruct/test/test_ieee.py
@@ -3,6 +3,7 @@
 import random
 import struct
 
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.rfloat import isnan, NAN, INFINITY
 from rpython.translator.c.test.test_genc import compile
@@ -73,9 +74,9 @@
                 assert repr(x) == repr(y), '%r != %r, Q=%r' % (x, y, Q)
 
         for be in [False, True]:
-            Q = []
-            ieee.pack_float(Q, x, 8, be)
-            Q = Q[0]
+            buf = MutableStringBuffer(8)
+            ieee.pack_float(buf, 0, x, 8, be)
+            Q = buf.finish()
             y = ieee.unpack_float(Q, be)
             assert repr(x) == repr(y), '%r != %r, Q=%r' % (x, y, Q)
 
@@ -197,12 +198,11 @@
 class TestCompiled:
     def test_pack_float(self):
         def pack(x, size):
-            result = []
-            ieee.pack_float(result, x, size, False)
+            buf = MutableStringBuffer(size)
+            ieee.pack_float(buf, 0, x, size, False)
             l = []
-            for x in result:
-                for c in x:
-                    l.append(str(ord(c)))
+            for c in buf.finish():
+                l.append(str(ord(c)))
             return ','.join(l)
         c_pack = compile(pack, [float, int])
 
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -15,6 +15,11 @@
     def advance(self, count):
         self.pos += count
 
+    def finish(self):
+        # check that we called advance() the right number of times
+        assert self.pos == self.result.getlength()
+        return self.result.finish()
+
     def _accept_arg(self):
         return self.value
 
@@ -41,7 +46,7 @@
         attrs = self.fmttable[fmt]
         pack = attrs['pack']
         pack(fake_fmtiter)
-        return fake_fmtiter.result.finish()
+        return fake_fmtiter.finish()
 
     def check(self, fmt, value):
         expected = struct.pack(self.endianess+fmt, value)

From pypy.commits at gmail.com  Wed May 10 19:05:44 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:44 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: test&fix native's pack_{float,
 double}. In the meantime,
 reduce some code duplication between nativefmttable.py and ieee.py
Message-ID: <59139cc8.5136c80a.ea1e.2150@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91233:6d4406afffe5
Date: 2017-05-10 19:04 +0200
http://bitbucket.org/pypy/pypy/changeset/6d4406afffe5/

Log:	test&fix native's pack_{float,double}. In the meantime, reduce some
	code duplication between nativefmttable.py and ieee.py

diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -264,9 +264,13 @@
     sign = r_ulonglong(sign)
     return (mant, (sign << BITS - MANT_DIG - 1) | exp)
 
- at jit.unroll_safe
+
 def pack_float(result, pos, x, size, be):
     unsigned = float_pack(x, size)
+    pack_float_to_buffer(result, pos, unsigned, size, be)
+
+ at jit.unroll_safe
+def pack_float_to_buffer(result, pos, unsigned, size, be):
     if be:
         # write in reversed order
         for i in range(size):
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -10,6 +10,7 @@
 from rpython.rlib.rstruct import standardfmttable as std
 from rpython.rlib.rstruct.standardfmttable import native_is_bigendian
 from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.rstruct.ieee import pack_float_to_buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
@@ -26,35 +27,19 @@
 # ____________________________________________________________
 
 
-range_8_unroll = unrolling_iterable(list(reversed(range(8))))
-range_4_unroll = unrolling_iterable(list(reversed(range(4))))
-
 def pack_double(fmtiter):
     doubleval = fmtiter.accept_float_arg()
     value = longlong2float.float2longlong(doubleval)
-    if fmtiter.bigendian:
-        for i in range_8_unroll:
-            x = (value >> (8*i)) & 0xff
-            fmtiter.result.append(chr(x))
-    else:
-        for i in range_8_unroll:
-            fmtiter.result.append(chr(value & 0xff))
-            value >>= 8
-
+    pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 8, fmtiter.bigendian)
+    fmtiter.advance(8)
 
 def pack_float(fmtiter):
     doubleval = fmtiter.accept_float_arg()
     floatval = r_singlefloat(doubleval)
     value = longlong2float.singlefloat2uint(floatval)
     value = widen(value)
-    if fmtiter.bigendian:
-        for i in range_4_unroll:
-            x = (value >> (8*i)) & 0xff
-            fmtiter.result.append(chr(x))
-    else:
-        for i in range_4_unroll:
-            fmtiter.result.append(chr(value & 0xff))
-            value >>= 8
+    pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 4, fmtiter.bigendian)
+    fmtiter.advance(4)
 
 # ____________________________________________________________
 #
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -1,5 +1,5 @@
 import pytest
-from rpython.rlib.rstruct import standardfmttable
+from rpython.rlib.rstruct import standardfmttable, nativefmttable
 from rpython.rlib.mutbuffer import MutableStringBuffer
 import struct
 
@@ -40,26 +40,25 @@
     implement struct.pack in pypy/module/struct
     """
 
-    endianess = None
-    fmttable = standardfmttable.standard_fmttable
+    bigendian = None
+    fmt_prefix = None
+    fmttable = None
 
     def mypack(self, fmt, value):
-        bigendian = self.endianess == '>'
         size = struct.calcsize(fmt)
-        fake_fmtiter = FakeFormatIter(bigendian, size, value)
+        fake_fmtiter = FakeFormatIter(self.bigendian, size, value)
         attrs = self.fmttable[fmt]
         pack = attrs['pack']
         pack(fake_fmtiter)
         return fake_fmtiter.finish()
 
     def mypack_fn(self, func, size, arg, value):
-        bigendian = self.endianess == '>'
-        fmtiter = FakeFormatIter(bigendian, size, value)
+        fmtiter = FakeFormatIter(self.bigendian, size, value)
         func(fmtiter, arg)
         return fmtiter.finish()
 
     def check(self, fmt, value):
-        expected = struct.pack(self.endianess+fmt, value)
+        expected = struct.pack(self.fmt_prefix+fmt, value)
         got = self.mypack(fmt, value)
         assert got == expected
 
@@ -108,8 +107,16 @@
 
 
 class TestPackLittleEndian(BaseTestPack):
-    endianess = '<'
-
+    bigendian = False
+    fmt_prefix = '<'
+    fmttable = standardfmttable.standard_fmttable
 
 class TestPackBigEndian(BaseTestPack):
-    endianess = '>'
+    bigendian = True
+    fmt_prefix = '>'
+    fmttable = standardfmttable.standard_fmttable
+
+class TestNative(BaseTestPack):
+    bigendian = nativefmttable.native_is_bigendian
+    fmt_prefix = '@'
+    fmttable = nativefmttable.native_fmttable

From pypy.commits at gmail.com  Wed May 10 19:05:46 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:46 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: adapt pypy/module/struct to
 the new interface for packing,
 which means to use MutableStringBuffer instead of StrinbBuilder everywhere
Message-ID: <59139cca.6e29c80a.e1e46.2022@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91234:b1e1e6b5e8bd
Date: 2017-05-11 00:46 +0200
http://bitbucket.org/pypy/pypy/changeset/b1e1e6b5e8bd/

Log:	adapt pypy/module/struct to the new interface for packing, which
	means to use MutableStringBuffer instead of StrinbBuilder everywhere

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -2,7 +2,7 @@
                                       maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 
@@ -14,7 +14,12 @@
         self.space = space
         self.args_w = args_w
         self.args_index = 0
-        self.result = StringBuilder(size)
+        self.pos = 0
+        self.result = MutableStringBuffer(size)
+        self.needs_zeros = False # MutableStringBuffer is already 0-inizialized
+
+    def advance(self, count):
+        self.pos += count
 
     # This *should* be always unroll safe, the only way to get here is by
     # unroll the interpret function, which means the fmt is const, and thus
@@ -31,8 +36,8 @@
 
     @jit.unroll_safe
     def align(self, mask):
-        pad = (-self.result.getlength()) & mask
-        self.result.append_multiple_char('\x00', pad)
+        pad = (-self.pos) & mask
+        self.advance(pad)
 
     def finished(self):
         if self.args_index != len(self.args_w):
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -40,10 +40,7 @@
 
 def _pack(space, format, args_w):
     """Return string containing values v1, v2, ... packed according to fmt."""
-    if jit.isconstant(format):
-        size = _calcsize(space, format)
-    else:
-        size = 8
+    size = _calcsize(space, format)
     fmtiter = PackFormatIterator(space, args_w, size)
     try:
         fmtiter.interpret(format)
@@ -51,7 +48,8 @@
         raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
         raise OperationError(get_error(space), space.newtext(e.msg))
-    return fmtiter.result.build()
+    assert fmtiter.pos == fmtiter.result.getlength(), 'missing .advance() or wrong calcsize()'
+    return fmtiter.result.finish()
 
 
 @unwrap_spec(format='text')
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,6 +4,7 @@
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs)
 from rpython.rlib.buffer import StringBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
@@ -84,10 +85,12 @@
 
     def readbuf_w(self, space):
         from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
-        builder = StringBuilder(len(self._value) * UNICODE_SIZE)
+        buf = MutableStringBuffer(len(self._value) * UNICODE_SIZE)
+        pos = 0
         for unich in self._value:
-            pack_unichar(unich, builder)
-        return StringBuffer(builder.build())
+            pack_unichar(unich, buf, pos)
+            pos += UNICODE_SIZE
+        return StringBuffer(buf.finish())
 
     def writebuf_w(self, space):
         raise oefmt(space.w_TypeError,
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -136,7 +136,8 @@
     if len(unistr) != 1:
         raise StructError("expected a unicode string of length 1")
     c = unistr[0]   # string->char conversion for the annotator
-    unichar.pack_unichar(c, fmtiter.result)
+    unichar.pack_unichar(c, fmtiter.result, fmtiter.pos)
+    fmtiter.advance(unichar.UNICODE_SIZE)
 
 @specialize.argtype(0)
 def unpack_unichar(fmtiter):
diff --git a/rpython/rlib/rstruct/unichar.py b/rpython/rlib/rstruct/unichar.py
--- a/rpython/rlib/rstruct/unichar.py
+++ b/rpython/rlib/rstruct/unichar.py
@@ -11,25 +11,25 @@
     UNICODE_SIZE = 4
 BIGENDIAN = sys.byteorder == "big"
 
-def pack_unichar(unich, charlist):
+def pack_unichar(unich, buf, pos):
     if UNICODE_SIZE == 2:
         if BIGENDIAN:
-            charlist.append(chr(ord(unich) >> 8))
-            charlist.append(chr(ord(unich) & 0xFF))
+            buf.setitem(pos,   chr(ord(unich) >> 8))
+            buf.setitem(pos+1, chr(ord(unich) & 0xFF))
         else:
-            charlist.append(chr(ord(unich) & 0xFF))
-            charlist.append(chr(ord(unich) >> 8))
+            buf.setitem(pos,   chr(ord(unich) & 0xFF))
+            buf.setitem(pos+1, chr(ord(unich) >> 8))
     else:
         if BIGENDIAN:
-            charlist.append(chr(ord(unich) >> 24))
-            charlist.append(chr((ord(unich) >> 16) & 0xFF))
-            charlist.append(chr((ord(unich) >> 8) & 0xFF))
-            charlist.append(chr(ord(unich) & 0xFF))
+            buf.setitem(pos,   chr(ord(unich) >> 24))
+            buf.setitem(pos+1, chr((ord(unich) >> 16) & 0xFF))
+            buf.setitem(pos+2, chr((ord(unich) >> 8) & 0xFF))
+            buf.setitem(pos+3, chr(ord(unich) & 0xFF))
         else:
-            charlist.append(chr(ord(unich) & 0xFF))
-            charlist.append(chr((ord(unich) >> 8) & 0xFF))
-            charlist.append(chr((ord(unich) >> 16) & 0xFF))
-            charlist.append(chr(ord(unich) >> 24))
+            buf.setitem(pos,   chr(ord(unich) & 0xFF))
+            buf.setitem(pos+1, chr((ord(unich) >> 8) & 0xFF))
+            buf.setitem(pos+2, chr((ord(unich) >> 16) & 0xFF))
+            buf.setitem(pos+3, chr(ord(unich) >> 24))
 
 def unpack_unichar(rawstring):
     assert len(rawstring) == UNICODE_SIZE

From pypy.commits at gmail.com  Wed May 10 19:05:49 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:49 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rpython fixes
Message-ID: <59139ccd.8f2fc80a.a9aaa.2215@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91235:db941580b490
Date: 2017-05-11 00:58 +0200
http://bitbucket.org/pypy/pypy/changeset/db941580b490/

Log:	rpython fixes

diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -267,18 +267,19 @@
 
 def pack_float(result, pos, x, size, be):
     unsigned = float_pack(x, size)
-    pack_float_to_buffer(result, pos, unsigned, size, be)
+    value = rarithmetic.longlongmask(unsigned)
+    pack_float_to_buffer(result, pos, value, size, be)
 
 @jit.unroll_safe
-def pack_float_to_buffer(result, pos, unsigned, size, be):
+def pack_float_to_buffer(result, pos, value, size, be):
     if be:
         # write in reversed order
         for i in range(size):
-            c = chr((unsigned >> (i * 8)) & 0xFF)
+            c = chr((value >> (i * 8)) & 0xFF)
             result.setitem(pos + size - i - 1, c)
     else:
         for i in range(size):
-            c = chr((unsigned >> (i * 8)) & 0xFF)
+            c = chr((value >> (i * 8)) & 0xFF)
             result.setitem(pos+i, c)
 
 @jit.unroll_safe
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -49,6 +49,7 @@
             for i in range(count - len(string)):
                 fmtiter.result.setitem(pos+i, '\x00')
     else:
+        assert count >= 0
         fmtiter.result.setslice(pos, string[:count])
     fmtiter.advance(count)
 

From pypy.commits at gmail.com  Wed May 10 19:05:52 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:52 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix marshal
Message-ID: <59139cd0.ce28c80a.7f98e.1e1f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91236:9142d98549d9
Date: 2017-05-11 01:03 +0200
http://bitbucket.org/pypy/pypy/changeset/9142d98549d9/

Log:	fix marshal

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -1,5 +1,5 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
 
@@ -191,9 +191,9 @@
 
 
 def pack_float(f):
-    result = StringBuilder(8)
-    ieee.pack_float(result, f, 8, False)
-    return result.build()
+    buf = MutableStringBuffer(8)
+    ieee.pack_float(buf, 0, f, 8, False)
+    return buf.finish()
 
 def unpack_float(s):
     return ieee.unpack_float(s, False)

From pypy.commits at gmail.com  Wed May 10 19:05:41 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 10 May 2017 16:05:41 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix pack_pascal
Message-ID: <59139cc5.4fd4370a.965c4.1d3f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91232:4c598c7f0b3e
Date: 2017-05-10 18:45 +0200
http://bitbucket.org/pypy/pypy/changeset/4c598c7f0b3e/

Log:	fix pack_pascal

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -40,18 +40,22 @@
     fmtiter.result.setitem(fmtiter.pos, c)
     fmtiter.advance(1)
 
-def pack_string(fmtiter, count):
+def _pack_string(fmtiter, string, count):
     pos = fmtiter.pos
-    string = fmtiter.accept_str_arg()
     if len(string) < count:
         fmtiter.result.setslice(pos, string)
         if fmtiter.needs_zeros:
-            for i in range(pos + len(string), count):
-                fmtiter.result.setitem(i, '\x00')
+            pos += len(string)
+            for i in range(count - len(string)):
+                fmtiter.result.setitem(pos+i, '\x00')
     else:
         fmtiter.result.setslice(pos, string[:count])
     fmtiter.advance(count)
 
+def pack_string(fmtiter, count):
+    string = fmtiter.accept_str_arg()
+    _pack_string(fmtiter, string, count)
+
 def pack_pascal(fmtiter, count):
     string = fmtiter.accept_str_arg()
     prefix = len(string)
@@ -60,12 +64,10 @@
         if prefix < 0:
             raise StructError("bad '0p' in struct format")
     if prefix > 255:
-        prefixchar = '\xff'
-    else:
-        prefixchar = chr(prefix)
-    fmtiter.result.append(prefixchar)
-    fmtiter.result.append_slice(string, 0, prefix)
-    fmtiter.result.append_multiple_char('\x00', count - (1 + prefix))
+        prefix = 255
+    fmtiter.result.setitem(fmtiter.pos, chr(prefix))
+    fmtiter.advance(1)
+    _pack_string(fmtiter, string, count-1)
 
 def make_float_packer(size):
     def packer(fmtiter):
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -102,10 +102,8 @@
         assert s == 'hello wo'
 
     def test_pack_pascal(self):
-        bigendian = self.endianess == '>'
-        fmtiter = FakeFormatIter(bigendian, 'hello')
-        standardfmttable.pack_pascal(fmtiter, 8)
-        s = fmtiter.result.build()
+        s = self.mypack_fn(standardfmttable.pack_pascal,
+                           arg=8, value='hello', size=8)
         assert s == '\x05hello\x00\x00'
 
 

From pypy.commits at gmail.com  Wed May 10 21:17:16 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 10 May 2017 18:17:16 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Restore py2 semantics for
 memoryview.__getitem__ and .__setitem__
Message-ID: <5913bb9c.ce28c80a.ed3db.0f83@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91237:d5640fcd6769
Date: 2017-05-11 02:16 +0100
http://bitbucket.org/pypy/pypy/changeset/d5640fcd6769/

Log:	Restore py2 semantics for memoryview.__getitem__ and .__setitem__

diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -1,4 +1,3 @@
-from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.buffer import StringBuffer, SubBuffer
 
 from pypy.interpreter.error import oefmt
@@ -69,18 +68,6 @@
         fmtiter.interpret(self.getformat())
         return fmtiter.result_w[0]
 
-    def bytes_from_value(self, space, w_val):
-        from pypy.module.struct.formatiterator import PackFormatIterator
-        itemsize = self.getitemsize()
-        fmtiter = PackFormatIterator(space, [w_val], itemsize)
-        try:
-            fmtiter.interpret(self.getformat())
-        except StructError as e:
-            raise oefmt(space.w_TypeError,
-                        "memoryview: invalid type for format '%s'",
-                        self.getformat())
-        return fmtiter.result.build()
-
     def _copy_buffer(self):
         if self.getndim() == 0:
             itemsize = self.getitemsize()
@@ -136,17 +123,11 @@
         itemsize = self.getitemsize()
         # TODO: this probably isn't very fast
         data = self.getbytes(offset, itemsize)
-        return self.value_from_bytes(space, data)
+        return space.newbytes(data)
 
     def new_slice(self, start, step, slicelength):
         return BufferSlice(self, start, step, slicelength)
 
-    def setitem_w(self, space, idx, w_obj):
-        offset = self.get_offset(space, 0, idx)
-        # TODO: this probably isn't very fast
-        byteval = self.bytes_from_value(space, w_obj)
-        self.setbytes(offset, byteval)
-
     def w_tolist(self, space):
         dim = self.getndim()
         if dim == 0:
@@ -253,10 +234,6 @@
         else:
             return BufferSlice(self, start, step, slicelength)
 
-    def setitem_w(self, space, idx, w_obj):
-        idx = self.get_offset(space, 0, idx)
-        self.data[idx] = space.byte_w(w_obj)
-
 
 class BufferSlice(BufferView):
     _immutable_ = True
@@ -316,6 +293,3 @@
         real_start = start + self.start
         real_step = self.step * step
         return BufferSlice(self.parent, real_start, real_step, slicelength)
-
-    def setitem_w(self, space, idx, w_obj):
-        return self.parent.setitem_w(space, self.parent_index(idx), w_obj)
diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -145,7 +145,11 @@
         start, stop, step, slicelength = self._decode_index(space, w_index, is_slice)
         itemsize = self.getitemsize()
         if step == 0:  # index only
-            self.view.setitem_w(space, start, w_obj)
+            value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+            if value.getitemsize() != itemsize:
+                raise oefmt(space.w_TypeError,
+                            "mismatching itemsizes for %T and %T", self, w_obj)
+            self.view.setbytes(start * itemsize, value.getbytes(0, itemsize))
         elif step == 1:
             value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
             if value.getlength() != slicelength * itemsize:
@@ -311,4 +315,4 @@
     elif (fort == 'A'):
         return (_IsCContiguous(ndim, shape, strides, itemsize) or
                 _IsFortranContiguous(ndim, shape, strides, itemsize))
-    return 0
\ No newline at end of file
+    return 0

From pypy.commits at gmail.com  Wed May 10 22:38:49 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 10 May 2017 19:38:49 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: cpyext fix
Message-ID: <5913ceb9.7633c80a.c4bea.19bd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91239:9257dbd0c043
Date: 2017-05-11 03:38 +0100
http://bitbucket.org/pypy/pypy/changeset/9257dbd0c043/

Log:	cpyext fix

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -469,10 +469,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self,
+        view = CPyBuffer(space, ptr[0], size, w_self,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return buf.wrap(space)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getwritebuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -487,10 +487,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+        view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return buf.wrap(space)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)

From pypy.commits at gmail.com  Wed May 10 22:38:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 10 May 2017 19:38:46 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Ensure that buffer() returns
 read-only buffers
Message-ID: <5913ceb6.6730ed0a.52b04.1b33@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91238:393498f6a662
Date: 2017-05-11 03:37 +0100
http://bitbucket.org/pypy/pypy/changeset/393498f6a662/

Log:	Ensure that buffer() returns read-only buffers

diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,7 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.buffer import SimpleView
+from pypy.interpreter.buffer import SimpleView, BufferInterfaceNotFound
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
@@ -40,7 +40,10 @@
     @staticmethod
     @unwrap_spec(offset=int, size=int)
     def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1):
-        buf = space.readbuf_w(w_object)
+        try:
+            buf = w_object.readbuf_w(space)
+        except BufferInterfaceNotFound:
+            raise oefmt(space.w_TypeError, "expected a readable buffer object")
         if offset == 0 and size == -1:
             return W_Buffer(buf)
         # handle buffer slices

From pypy.commits at gmail.com  Thu May 11 05:03:54 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 02:03:54 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: I don't know why this popped
 out now as it seems unrelated with my recent changes:
 scoped_nonmovingbuffer expects data to be non-null,
 so we enforce the annotation to get an early error. Then,
 we fix the problem in interp_bz2.py
Message-ID: <591428fa.4468370a.4398a.3966@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91240:beae5a5ccb0b
Date: 2017-05-11 10:03 +0100
http://bitbucket.org/pypy/pypy/changeset/beae5a5ccb0b/

Log:	I don't know why this popped out now as it seems unrelated with my
	recent changes: scoped_nonmovingbuffer expects data to be non-null,
	so we enforce the annotation to get an early error. Then, we fix the
	problem in interp_bz2.py

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -1,4 +1,5 @@
 from __future__ import with_statement
+from rpython.annotator import model as annmodel
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
@@ -552,6 +553,7 @@
         to compress, call the flush() method to finish the compression process,
         and return what is left in the internal buffers."""
 
+        assert data is not None
         datasize = len(data)
 
         if datasize == 0:
@@ -662,6 +664,7 @@
         was found after the end of stream, it'll be ignored and saved in
         unused_data attribute."""
 
+        assert data is not None
         if not self.running:
             raise oefmt(self.space.w_EOFError,
                         "end of stream was already found")
@@ -715,6 +718,7 @@
     use an instance of BZ2Compressor instead. The compresslevel parameter, if
     given, must be a number between 1 and 9."""
 
+    assert data is not None
     if compresslevel < 1 or compresslevel > 9:
         raise oefmt(space.w_ValueError,
                     "compresslevel must be between 1 and 9")
@@ -757,6 +761,7 @@
     Decompress data in one shot. If you want to decompress data sequentially,
     use an instance of BZ2Decompressor instead."""
 
+    assert data is not None
     in_bufsize = len(data)
     if in_bufsize == 0:
         return space.newbytes("")
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -1232,8 +1232,11 @@
 
 
 class scoped_nonmovingbuffer:
+
     def __init__(self, data):
         self.data = data
+    __init__._annenforceargs_ = [None, annmodel.SomeString(can_be_None=False)]
+
     def __enter__(self):
         self.buf, self.flag = get_nonmovingbuffer(self.data)
         return self.buf

From pypy.commits at gmail.com  Thu May 11 05:23:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 May 2017 02:23:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: Rewrite itertools.groupby(),
 following CPython instead of having many
Message-ID: <59142d99.0951240a.bd55.fb1e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91241:6093ff1a44e6
Date: 2017-05-11 11:22 +0200
http://bitbucket.org/pypy/pypy/changeset/6093ff1a44e6/

Log:	Rewrite itertools.groupby(), following CPython instead of having
	many flags to get a result that differs subtly

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -920,90 +920,42 @@
 class W_GroupBy(W_Root):
     def __init__(self, space, w_iterable, w_fun):
         self.space = space
-        self.w_iterable = self.space.iter(w_iterable)
-        if space.is_none(w_fun):
-            self.w_fun = None
-        else:
-            self.w_fun = w_fun
-        self.index = 0
-        self.lookahead = False
-        self.exhausted = False
-        self.started = False
-        # new_group - new group not started yet, next should not skip any items
-        self.new_group = True
-        self.w_lookahead = self.space.w_None
-        self.w_key = self.space.w_None
+        self.w_iterator = self.space.iter(w_iterable)
+        if w_fun is None:
+            w_fun = space.w_None
+        self.w_keyfunc = w_fun
+        self.w_tgtkey = None
+        self.w_currkey = None
+        self.w_currvalue = None
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        self._skip_to_next_iteration_group()
+        w_key = self.w_tgtkey = self.w_currkey
+        w_grouper = W_GroupByIterator(self, w_key)
+        return self.space.newtuple([w_key, w_grouper])
 
-        if not self.new_group:
-            self._consume_unwanted_input()
+    def _skip_to_next_iteration_group(self):
+        space = self.space
+        while True:
+            if self.w_currkey is None:
+                pass
+            elif self.w_tgtkey is None:
+                break
+            else:
+                if not space.eq_w(self.w_tgtkey, self.w_currkey):
+                    break
 
-        if not self.started:
-            self.started = True
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                raise
+            w_newvalue = space.next(self.w_iterator)
+            if space.is_w(self.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
             else:
-                self.w_lookahead = w_obj
-                if self.w_fun is None:
-                    self.w_key = w_obj
-                else:
-                    self.w_key = self.space.call_function(self.w_fun, w_obj)
-                self.lookahead = True
+                w_newkey = space.call_function(self.w_keyfunc, w_newvalue)
 
-        self.new_group = False
-        w_iterator = W_GroupByIterator(self.space, self.index, self)
-        return self.space.newtuple([self.w_key, w_iterator])
-
-    def _consume_unwanted_input(self):
-        # Consume unwanted input until we reach the next group
-        try:
-            while True:
-                self.group_next(self.index)
-        except StopIteration:
-            pass
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
-    def group_next(self, group_index):
-        if group_index < self.index:
-            raise StopIteration
-        else:
-            if self.lookahead:
-                self.lookahead = False
-                return self.w_lookahead
-
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                    raise StopIteration
-                else:
-                    raise
-            else:
-                if self.w_fun is None:
-                    w_new_key = w_obj
-                else:
-                    w_new_key = self.space.call_function(self.w_fun, w_obj)
-                if self.space.eq_w(self.w_key, w_new_key):
-                    return w_obj
-                else:
-                    self.index += 1
-                    self.w_lookahead = w_obj
-                    self.w_key = w_new_key
-                    self.lookahead = True
-                    self.new_group = True #new group
-                    raise StopIteration
+            self.w_currkey = w_newkey
+            self.w_currvalue = w_newvalue
 
 def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
     r = space.allocate_instance(W_GroupBy, w_subtype)
@@ -1036,26 +988,33 @@
 
 
 class W_GroupByIterator(W_Root):
-    def __init__(self, space, index, groupby):
-        self.space = space
-        self.index = index
+    def __init__(self, groupby, w_tgtkey):
         self.groupby = groupby
-        self.exhausted = False
+        self.w_tgtkey = w_tgtkey
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        groupby = self.groupby
+        space = groupby.space
+        if groupby.w_currvalue is None:
+            w_newvalue = space.next(groupby.w_iterator)
+            if space.is_w(groupby.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
+            else:
+                w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
+            assert groupby.w_currvalue is None
+            groupby.w_currkey = w_newkey
+            groupby.w_currvalue = w_newvalue
 
-        try:
-            w_obj = self.groupby.group_next(self.index)
-        except StopIteration:
-            self.exhausted = True
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        else:
-            return w_obj
+        assert groupby.w_currkey is not None
+        if not space.eq_w(self.w_tgtkey, groupby.w_currkey):
+            raise OperationError(space.w_StopIteration, space.w_None)
+        w_result = groupby.w_currvalue
+        groupby.w_currvalue = None
+        groupby.w_currkey = None
+        return w_result
 
 W_GroupByIterator.typedef = TypeDef(
         'itertools._groupby',
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -634,6 +634,17 @@
         it = itertools.groupby([0], 1)
         raises(TypeError, it.next)
 
+    def test_groupby_question_43905804(self):
+        # http://stackoverflow.com/questions/43905804/
+        import itertools
+
+        inputs = ((x > 5, x) for x in range(10))
+        (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+        a = list(a)
+        b = list(b)
+        assert a == []
+        assert b == [(True, 9)]
+
     def test_iterables(self):
         import itertools
     

From pypy.commits at gmail.com  Thu May 11 05:39:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 May 2017 02:39:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Manual merge of 6093ff1a44e6,
 plus reduce/setstate
Message-ID: <5914313a.4fd4370a.bdd07.402d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91242:cd0822695fae
Date: 2017-05-11 11:38 +0200
http://bitbucket.org/pypy/pypy/changeset/cd0822695fae/

Log:	Manual merge of 6093ff1a44e6, plus reduce/setstate

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -971,109 +971,57 @@
 class W_GroupBy(W_Root):
     def __init__(self, space, w_iterable, w_fun):
         self.space = space
-        self.w_iterable = self.space.iter(w_iterable)
-        if space.is_none(w_fun):
-            self.w_fun = None
-        else:
-            self.w_fun = w_fun
-        self.index = 0
-        self.lookahead = False
-        self.exhausted = False
-        self.started = False
-        # new_group - new group not started yet, next should not skip any items
-        self.new_group = True
-        self.w_lookahead = self.space.w_None
-        self.w_key = self.space.w_None
+        self.w_iterator = self.space.iter(w_iterable)
+        if w_fun is None:
+            w_fun = space.w_None
+        self.w_keyfunc = w_fun
+        self.w_tgtkey = None
+        self.w_currkey = None
+        self.w_currvalue = None
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        self._skip_to_next_iteration_group()
+        w_key = self.w_tgtkey = self.w_currkey
+        w_grouper = W_GroupByIterator(self, w_key)
+        return self.space.newtuple([w_key, w_grouper])
 
-        if not self.new_group:
-            self._consume_unwanted_input()
+    def _skip_to_next_iteration_group(self):
+        space = self.space
+        while True:
+            if self.w_currkey is None:
+                pass
+            elif self.w_tgtkey is None:
+                break
+            else:
+                if not space.eq_w(self.w_tgtkey, self.w_currkey):
+                    break
 
-        if not self.started:
-            self.started = True
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                raise
+            w_newvalue = space.next(self.w_iterator)
+            if space.is_w(self.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
             else:
-                self.w_lookahead = w_obj
-                if self.w_fun is None:
-                    self.w_key = w_obj
-                else:
-                    self.w_key = self.space.call_function(self.w_fun, w_obj)
-                self.lookahead = True
+                w_newkey = space.call_function(self.w_keyfunc, w_newvalue)
 
-        self.new_group = False
-        w_iterator = W_GroupByIterator(self.space, self.index, self)
-        return self.space.newtuple([self.w_key, w_iterator])
-
-    def _consume_unwanted_input(self):
-        # Consume unwanted input until we reach the next group
-        try:
-            while True:
-                self.group_next(self.index)
-        except StopIteration:
-            pass
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
-    def group_next(self, group_index):
-        if group_index < self.index:
-            raise StopIteration
-        else:
-            if self.lookahead:
-                self.lookahead = False
-                return self.w_lookahead
-
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                    raise StopIteration
-                else:
-                    raise
-            else:
-                if self.w_fun is None:
-                    w_new_key = w_obj
-                else:
-                    w_new_key = self.space.call_function(self.w_fun, w_obj)
-                if self.space.eq_w(self.w_key, w_new_key):
-                    return w_obj
-                else:
-                    self.index += 1
-                    self.w_lookahead = w_obj
-                    self.w_key = w_new_key
-                    self.lookahead = True
-                    self.new_group = True #new group
-                    raise StopIteration
+            self.w_currkey = w_newkey
+            self.w_currvalue = w_newvalue
 
     def descr_reduce(self, space):
-        if self.started:
-            return space.newtuple([
-                space.type(self),
+        items_w = [space.type(self),
+                   space.newtuple([
+                       self.w_iterator,
+                       self.w_keyfunc])]
+        if (self.w_tgtkey is not None and self.w_currkey is not None
+                and self.w_currvalue is not None):
+            items_w = items_w + [
                 space.newtuple([
-                    self.w_iterable,
-                    self.w_fun]),
-                space.newtuple([
-                    self.w_key,
-                    self.w_lookahead,
-                    self.w_key])
-                ])
-        else:
-            return space.newtuple([
-                space.type(self),
-                space.newtuple([
-                    self.w_iterable,
-                    self.w_fun])])
+                    self.w_currkey,
+                    self.w_currvalue,
+                    self.w_tgtkey])
+                ]
+        return space.newtuple(items_w)
 
     def descr_setstate(self, space, w_state):
         state = space.unpackiterable(w_state)
@@ -1082,12 +1030,7 @@
             raise oefmt(space.w_TypeError,
                         "function takes exactly 3 arguments (%d given)",
                         num_args)
-        w_key, w_lookahead, _ = state
-        self.w_key = w_key
-        self.w_lookahead = w_lookahead
-        if self.w_lookahead:
-            self.started = True
-            self.lookahead = True
+        self.w_currkey, self.w_currvalue, self.w_tgtkey = state
 
 def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
     r = space.allocate_instance(W_GroupBy, w_subtype)
@@ -1122,40 +1065,46 @@
 
 
 class W_GroupByIterator(W_Root):
-    def __init__(self, space, index, groupby):
-        self.space = space
-        self.index = index
+    def __init__(self, groupby, w_tgtkey):
         self.groupby = groupby
-        self.exhausted = False
+        self.w_tgtkey = w_tgtkey
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        groupby = self.groupby
+        space = groupby.space
+        if groupby.w_currvalue is None:
+            w_newvalue = space.next(groupby.w_iterator)
+            if space.is_w(groupby.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
+            else:
+                w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
+            assert groupby.w_currvalue is None
+            groupby.w_currkey = w_newkey
+            groupby.w_currvalue = w_newvalue
 
-        try:
-            w_obj = self.groupby.group_next(self.index)
-        except StopIteration:
-            self.exhausted = True
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        else:
-            return w_obj
+        assert groupby.w_currkey is not None
+        if not space.eq_w(self.w_tgtkey, groupby.w_currkey):
+            raise OperationError(space.w_StopIteration, space.w_None)
+        w_result = groupby.w_currvalue
+        groupby.w_currvalue = None
+        groupby.w_currkey = None
+        return w_result
 
     def descr_reduce(self, space):
         return space.newtuple([
             space.type(self),
             space.newtuple([
-                space.newint(self.index),
-                self.groupby]),
+                self.groupby,
+                self.w_tgtkey]),
             ])
 
-def W_GroupByIterator__new__(space, w_subtype, w_index, w_groupby):
+def W_GroupByIterator__new__(space, w_subtype, w_parent, w_tgtkey):
     r = space.allocate_instance(W_GroupByIterator, w_subtype)
-    index = space.int_w(w_index)
-    groupby = space.interp_w(W_GroupBy, w_groupby)
-    r.__init__(space, index, groupby)
+    groupby = space.interp_w(W_GroupBy, w_parent)
+    r.__init__(groupby, w_tgtkey)
     return r
 
 W_GroupByIterator.typedef = TypeDef(
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -516,6 +516,17 @@
         it = itertools.groupby([0], 1)
         raises(TypeError, next, it)
 
+    def test_groupby_question_43905804(self):
+        # http://stackoverflow.com/questions/43905804/
+        import itertools
+
+        inputs = ((x > 5, x) for x in range(10))
+        (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+        a = list(a)
+        b = list(b)
+        assert a == []
+        assert b == [(True, 9)]
+
     def test_iterables(self):
         import itertools
 

From pypy.commits at gmail.com  Thu May 11 05:52:07 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 02:52:07 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: we cannot malloc(STR,
 zero=True) because the GC does not support it. So, simplify a bit the code,
 remove the needs_zeros flag and always write zeros when it's needed
Message-ID: <59143447.e43aed0a.f56f.3dde@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91243:582e284508de
Date: 2017-05-11 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/582e284508de/

Log:	we cannot malloc(STR, zero=True) because the GC does not support it.
	So, simplify a bit the code, remove the needs_zeros flag and always
	write zeros when it's needed

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -16,7 +16,6 @@
         self.args_index = 0
         self.pos = 0
         self.result = MutableStringBuffer(size)
-        self.needs_zeros = False # MutableStringBuffer is already 0-inizialized
 
     def advance(self, count):
         self.pos += count
@@ -37,6 +36,7 @@
     @jit.unroll_safe
     def align(self, mask):
         pad = (-self.pos) & mask
+        self.result.setzeros(self.pos, pad)
         self.advance(pad)
 
     def finished(self):
diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -1,6 +1,6 @@
-from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
 from rpython.rtyper.annlowlevel import llstr, hlstr
 from rpython.rlib.buffer import Buffer
 
@@ -21,10 +21,8 @@
 
     def __init__(self, size):
         self.readonly = False
-        # rstr.mallocstr does not pass zero=True, so we call lltype.malloc
-        # directly
         self.size = size
-        self.ll_val = lltype.malloc(STR, size, zero=True)
+        self.ll_val = mallocstr(size)
 
     def getlength(self):
         return self.size
@@ -40,5 +38,13 @@
     def as_str(self):
         raise ValueError('as_str() is not supported. Use finish() instead')
 
+    def _hlstr(self):
+        assert not we_are_translated() # debug only
+        return hlstr(self.ll_val)
+
     def setitem(self, index, char):
         self.ll_val.chars[index] = char
+
+    def setzeros(self, index, count):
+        for i in range(index, index+count):
+            self.setitem(i, '\x00')
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -21,10 +21,7 @@
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
 
 def pack_pad(fmtiter, count):
-    if fmtiter.needs_zeros:
-        pos = fmtiter.pos
-        for i in range(count):
-            fmtiter.result.setitem(pos+i, '\x00')
+    fmtiter.result.setzeros(fmtiter.pos, count)
     fmtiter.advance(count)
 
 def pack_char(fmtiter):
@@ -43,11 +40,9 @@
 def _pack_string(fmtiter, string, count):
     pos = fmtiter.pos
     if len(string) < count:
+        n = len(string)
         fmtiter.result.setslice(pos, string)
-        if fmtiter.needs_zeros:
-            pos += len(string)
-            for i in range(count - len(string)):
-                fmtiter.result.setitem(pos+i, '\x00')
+        fmtiter.result.setzeros(pos+n, count-n)
     else:
         assert count >= 0
         fmtiter.result.setslice(pos, string[:count])
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -10,11 +10,7 @@
         self.value = value
         self.bigendian = bigendian
         self.result = MutableStringBuffer(size)
-        # we set the buffer to non-zero, so ensure that we actively write 0s
-        # where it's needed
-        self.result.setslice(0, '\xff'*size)
         self.pos = 0
-        self.needs_zeros = True
 
     def advance(self, count):
         self.pos += count
diff --git a/rpython/rlib/test/test_mutbuffer.py b/rpython/rlib/test/test_mutbuffer.py
--- a/rpython/rlib/test/test_mutbuffer.py
+++ b/rpython/rlib/test/test_mutbuffer.py
@@ -22,3 +22,9 @@
         buf = MutableStringBuffer(6)
         buf.setslice(2, 'ABCD')
         assert buf.finish() == '\x00\x00ABCD'
+
+    def test_setzeros(self):
+        buf = MutableStringBuffer(8)
+        buf.setslice(0, 'ABCDEFGH')
+        buf.setzeros(2, 3)
+        assert buf.finish() == 'AB\x00\x00\x00FGH'

From pypy.commits at gmail.com  Thu May 11 05:52:10 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 02:52:10 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: remove the last remainder of
 the killed strstorage
Message-ID: <5914344a.013ced0a.fd900.3d71@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91244:a12006f0de48
Date: 2017-05-11 11:29 +0200
http://bitbucket.org/pypy/pypy/changeset/a12006f0de48/

Log:	remove the last remainder of the killed strstorage

diff --git a/rpython/jit/backend/x86/test/test_strstorage.py b/rpython/jit/backend/x86/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_strstorage.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.metainterp.test.test_strstorage import TestStrStorage as _TestStrStorage
-
-
-class TestStrStorage(Jit386Mixin, _TestStrStorage):
-    # for the individual tests see
-    # ====> ../../../metainterp/test/test_strstorage.py
-    pass

From pypy.commits at gmail.com  Thu May 11 06:10:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 May 2017 03:10:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: in theory, this is a fix
Message-ID: <591438a6.5919370a.3f38d.4346@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91245:5ebcb8f995fd
Date: 2017-05-11 12:10 +0200
http://bitbucket.org/pypy/pypy/changeset/5ebcb8f995fd/

Log:	in theory, this is a fix

diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -625,7 +625,7 @@
     # use r_uint to perform a single comparison (this whole function is
     # getting inlined into every caller so keeping the branching to a
     # minimum is a good idea)
-    index = r_uint(x - lower)
+    index = r_uint(x) - r_uint(lower)
     if index >= r_uint(upper - lower):
         w_res = instantiate(W_IntObject)
     else:

From pypy.commits at gmail.com  Thu May 11 06:40:15 2017
From: pypy.commits at gmail.com (sthalik)
Date: Thu, 11 May 2017 03:40:15 -0700 (PDT)
Subject: [pypy-commit] pypy sthalik/fix-signed-integer-sizes-1494493539409:
 fix signed integer sizes
Message-ID: <59143f8f.0538c80a.6fc8.43e4@mx.google.com>

Author: Stanis&#322;aw Halik <sthalik at misaki.pl>
Branch: sthalik/fix-signed-integer-sizes-1494493539409
Changeset: r91246:cb8ca968878f
Date: 2017-05-11 09:06 +0000
http://bitbucket.org/pypy/pypy/changeset/cb8ca968878f/

Log:	fix signed integer sizes

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -343,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision

From pypy.commits at gmail.com  Thu May 11 06:40:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 May 2017 03:40:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 sthalik/pypy-1/sthalik/fix-signed-integer-sizes-1494493539409 (pull request
 #543)
Message-ID: <59143f92.c15b370a.79d40.4354@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r91247:400ffc527c25
Date: 2017-05-11 10:39 +0000
http://bitbucket.org/pypy/pypy/changeset/400ffc527c25/

Log:	Merged in sthalik/pypy-1/sthalik/fix-signed-integer-
	sizes-1494493539409 (pull request #543)

	fix signed integer sizes

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -343,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision

From pypy.commits at gmail.com  Thu May 11 10:51:58 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 07:51:58 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: deleting strstorage tests was
 a mistake,
 because they were the only ones to test llop.gc_load_indexed. Rewrite them
 in a more direct way
Message-ID: <59147a8e.50b5500a.e5001.1963@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91248:22c587e1da2b
Date: 2017-05-11 16:09 +0200
http://bitbucket.org/pypy/pypy/changeset/22c587e1da2b/

Log:	deleting strstorage tests was a mistake, because they were the only
	ones to test llop.gc_load_indexed. Rewrite them in a more direct way

diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/rtyper/test/test_llop.py
@@ -0,0 +1,46 @@
+import struct
+from rpython.rtyper.test.tool import BaseRtypingTest
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib.rarithmetic import r_singlefloat
+
+def str_offset():
+    base_ofs = (llmemory.offsetof(STR, 'chars') +
+                llmemory.itemoffsetof(STR.chars, 0))
+    scale_factor = llmemory.sizeof(lltype.Char)
+    return base_ofs, scale_factor
+
+class BaseLLOpTest(object):
+    
+    def test_gc_load_indexed(self):
+        buf = struct.pack('dfi', 123.456, 123.456, 0x12345678)
+        val = self.gc_load_from_string(rffi.DOUBLE, buf, 0)
+        assert val == 123.456
+        #
+        val = self.gc_load_from_string(rffi.FLOAT, buf, 8)
+        assert val == r_singlefloat(123.456)
+        #
+        val = self.gc_load_from_string(rffi.INT, buf, 12)
+        assert val == 0x12345678
+
+
+class TestDirect(BaseLLOpTest):
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        base_ofs, scale_factor = str_offset()
+        lls = llstr(buf)
+        return llop.gc_load_indexed(TYPE, lls, offset,
+                                    scale_factor, base_ofs)
+
+
+class TestRTyping(BaseLLOpTest, BaseRtypingTest):
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        def fn(offset):
+            lls = llstr(buf)
+            base_ofs, scale_factor = str_offset()
+            return llop.gc_load_indexed(TYPE, lls, offset,
+                                        scale_factor, base_ofs)
+        return self.interpret(fn, [offset])
diff --git a/rpython/translator/c/test/test_llop.py b/rpython/translator/c/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/translator/c/test/test_llop.py
@@ -0,0 +1,31 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rtyper.test.test_llop import BaseLLOpTest, str_offset
+from rpython.translator.c.test.test_genc import compile
+
+
+class TestLLOp(BaseLLOpTest):
+    cache = {}
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        if TYPE not in self.cache:
+            assert isinstance(TYPE, lltype.Primitive)
+            if TYPE in (lltype.Float, lltype.SingleFloat):
+                TARGET_TYPE = lltype.Float
+            else:
+                TARGET_TYPE = lltype.Signed
+
+            def llf(buf, offset):
+                base_ofs, scale_factor = str_offset()
+                lls = llstr(buf)
+                x = llop.gc_load_indexed(TYPE, lls, offset,
+                                         scale_factor, base_ofs)
+                return lltype.cast_primitive(TARGET_TYPE, x)
+
+            fn = compile(llf, [str, int])
+            self.cache[TYPE] = fn
+        #
+        fn = self.cache[TYPE]
+        x = fn(buf, offset)
+        return lltype.cast_primitive(TYPE, x)

From pypy.commits at gmail.com  Thu May 11 10:52:02 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 07:52:02 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add the JIT version of
 test_llop
Message-ID: <59147a92.2db3500a.52435.3890@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91250:892ab4160ec6
Date: 2017-05-11 16:21 +0200
http://bitbucket.org/pypy/pypy/changeset/892ab4160ec6/

Log:	add the JIT version of test_llop

diff --git a/rpython/jit/backend/x86/test/test_llop.py b/rpython/jit/backend/x86/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_llop.py
@@ -0,0 +1,9 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(Jit386Mixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+    pass
+
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -0,0 +1,49 @@
+import py
+import sys
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.test_llop import BaseLLOpTest, str_gc_load
+from rpython.jit.codewriter import longlong
+from rpython.jit.metainterp.history import getkind
+from rpython.jit.metainterp.test.support import LLJitMixin
+
+class TestLLOp(BaseLLOpTest, LLJitMixin):
+
+    # for the individual tests see
+    # ====> ../../../rtyper/test/test_llop.py
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        def f(offset):
+            return str_gc_load(TYPE, buf, offset)
+        res = self.interp_operations(f, [offset], supports_singlefloats=True)
+        #
+        kind = getkind(TYPE)[0] # 'i' or 'f'
+        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
+                                       'finish': 1})
+        #
+        if TYPE == lltype.SingleFloat:
+            # interp_operations returns the int version of r_singlefloat, but
+            # our tests expects to receive an r_singlefloat: let's convert it
+            # back!
+            return longlong.int2singlefloat(res)
+        return res
+
+    def test_force_virtual_str_storage(self):
+        byteorder = sys.byteorder
+        size = rffi.sizeof(lltype.Signed)
+        def f(val):
+            if byteorder == 'little':
+                x = chr(val) + '\x00'*(size-1)
+            else:
+                x = '\x00'*(size-1) + chr(val)
+            return str_gc_load(lltype.Signed, x, 0)
+        res = self.interp_operations(f, [42], supports_singlefloats=True)
+        assert res == 42
+        self.check_operations_history({
+            'newstr': 1,              # str forcing
+            'strsetitem': 1,          # str forcing
+            'call_pure_r': 1,         # str forcing (copystrcontent)
+            'guard_no_exception': 1,  # str forcing
+            'gc_load_indexed_i': 1,   # str_storage_getitem
+            'finish': 1
+            })

From pypy.commits at gmail.com  Thu May 11 10:52:00 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 11 May 2017 07:52:00 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: reduce code duplication
Message-ID: <59147a90.6997500a.fe174.1865@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91249:be9e1d6fe67b
Date: 2017-05-11 16:14 +0200
http://bitbucket.org/pypy/pypy/changeset/be9e1d6fe67b/

Log:	reduce code duplication

diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -6,11 +6,13 @@
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rlib.rarithmetic import r_singlefloat
 
-def str_offset():
+def str_gc_load(TYPE, buf, offset):
     base_ofs = (llmemory.offsetof(STR, 'chars') +
                 llmemory.itemoffsetof(STR.chars, 0))
     scale_factor = llmemory.sizeof(lltype.Char)
-    return base_ofs, scale_factor
+    lls = llstr(buf)
+    return llop.gc_load_indexed(TYPE, lls, offset,
+                                scale_factor, base_ofs)
 
 class BaseLLOpTest(object):
     
@@ -29,18 +31,11 @@
 class TestDirect(BaseLLOpTest):
 
     def gc_load_from_string(self, TYPE, buf, offset):
-        base_ofs, scale_factor = str_offset()
-        lls = llstr(buf)
-        return llop.gc_load_indexed(TYPE, lls, offset,
-                                    scale_factor, base_ofs)
-
+        return str_gc_load(TYPE, buf, offset)
 
 class TestRTyping(BaseLLOpTest, BaseRtypingTest):
 
     def gc_load_from_string(self, TYPE, buf, offset):
         def fn(offset):
-            lls = llstr(buf)
-            base_ofs, scale_factor = str_offset()
-            return llop.gc_load_indexed(TYPE, lls, offset,
-                                        scale_factor, base_ofs)
+            return str_gc_load(TYPE, buf, offset)
         return self.interpret(fn, [offset])
diff --git a/rpython/translator/c/test/test_llop.py b/rpython/translator/c/test/test_llop.py
--- a/rpython/translator/c/test/test_llop.py
+++ b/rpython/translator/c/test/test_llop.py
@@ -1,7 +1,5 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rtyper.lltypesystem.lloperation import llop
-from rpython.rtyper.annlowlevel import llstr
-from rpython.rtyper.test.test_llop import BaseLLOpTest, str_offset
+from rpython.rtyper.test.test_llop import BaseLLOpTest, str_gc_load
 from rpython.translator.c.test.test_genc import compile
 
 
@@ -17,10 +15,7 @@
                 TARGET_TYPE = lltype.Signed
 
             def llf(buf, offset):
-                base_ofs, scale_factor = str_offset()
-                lls = llstr(buf)
-                x = llop.gc_load_indexed(TYPE, lls, offset,
-                                         scale_factor, base_ofs)
+                x = str_gc_load(TYPE, buf, offset)
                 return lltype.cast_primitive(TARGET_TYPE, x)
 
             fn = compile(llf, [str, int])

From pypy.commits at gmail.com  Thu May 11 11:44:38 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 May 2017 08:44:38 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Fix memoryview.__setitem__,
 again
Message-ID: <591486e6.5ba5500a.6c4df.2ca7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91251:3ebd85b5b58e
Date: 2017-05-11 16:02 +0100
http://bitbucket.org/pypy/pypy/changeset/3ebd85b5b58e/

Log:	Fix memoryview.__setitem__, again

diff --git a/pypy/objspace/std/memoryobject.py b/pypy/objspace/std/memoryobject.py
--- a/pypy/objspace/std/memoryobject.py
+++ b/pypy/objspace/std/memoryobject.py
@@ -144,18 +144,11 @@
         is_slice = space.isinstance_w(w_index, space.w_slice)
         start, stop, step, slicelength = self._decode_index(space, w_index, is_slice)
         itemsize = self.getitemsize()
-        if step == 0:  # index only
-            value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getitemsize() != itemsize:
-                raise oefmt(space.w_TypeError,
-                            "mismatching itemsizes for %T and %T", self, w_obj)
-            self.view.setbytes(start * itemsize, value.getbytes(0, itemsize))
-        elif step == 1:
-            value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
-            if value.getlength() != slicelength * itemsize:
-                raise oefmt(space.w_ValueError,
-                            "cannot modify size of memoryview object")
-            self.view.setbytes(start * itemsize, value.as_str())
+        value = space.buffer_w(w_obj, space.BUF_CONTIG_RO)
+        if value.getlength() != slicelength * itemsize:
+            raise oefmt(space.w_ValueError,
+                        "cannot modify size of memoryview object")
+        self.view.setbytes(start * itemsize, value.as_str())
 
     def descr_len(self, space):
         self._check_released(space)

From pypy.commits at gmail.com  Thu May 11 11:44:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 May 2017 08:44:40 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: fix _rawffi
Message-ID: <591486e8.4288500a.6300c.1721@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91252:4574a68bea35
Date: 2017-05-11 16:09 +0100
http://bitbucket.org/pypy/pypy/changeset/4574a68bea35/

Log:	fix _rawffi

diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -374,6 +374,12 @@
     def buffer_w(self, space, flags):
         return SimpleView(RawFFIBuffer(self))
 
+    def readbuf_w(self, space):
+        return RawFFIBuffer(self)
+
+    def writebuf_w(self, space):
+        return RawFFIBuffer(self)
+
     def getrawsize(self):
         raise NotImplementedError("abstract base class")
 

From pypy.commits at gmail.com  Thu May 11 11:44:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 May 2017 08:44:43 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: fix micronumpy
Message-ID: <591486eb.bbad500a.e0da1.1dea@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91253:2372a325f694
Date: 2017-05-11 16:43 +0100
http://bitbucket.org/pypy/pypy/changeset/2372a325f694/

Log:	fix micronumpy

diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -762,3 +762,6 @@
 
     def getstrides(self):
         return self.impl.strides
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -91,7 +91,7 @@
             w_base = w_object
             if read_only:
                 w_base = None
-            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data,
                                 dtype, w_base=w_base, strides=strides,
                                 start=offset), read_only
         if w_data is None:
@@ -104,11 +104,11 @@
         #print 'create view from shape',shape,'dtype',dtype,'data',data
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
-                   "__array_interface__ strides not fully supported yet") 
+                   "__array_interface__ strides not fully supported yet")
         arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
-        
+
     except OperationError as e:
         if e.match(space, space.w_AttributeError):
             return None, False
@@ -120,7 +120,7 @@
         return descr
     msg = "invalid PEP 3118 format string: '%s'" % c_format
     space.warn(space.newtext(msg), space.w_RuntimeWarning)
-    return None 
+    return None
 
 def _array_from_buffer_3118(space, w_object, dtype):
     try:
@@ -139,12 +139,12 @@
             raise oefmt(space.w_NotImplementedError,
                 "creating an array from a memoryview while specifying dtype "
                 "not supported")
-        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))):
             msg = ("Item size computed from the PEP 3118 buffer format "
                   "string does not match the actual item size.")
             space.warn(space.newtext(msg), space.w_RuntimeWarning)
             return w_object
-        dtype = descr 
+        dtype = descr
     elif not dtype:
         dtype = descriptor.get_dtype_cache(space).w_stringdtype
         dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
@@ -181,7 +181,7 @@
             raise e
     writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
     w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
-               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               storage_bytes=buflen, dtype=dtype, w_base=w_object,
                writable=writable, strides=strides)
     if w_ret:
         return w_ret
@@ -212,7 +212,7 @@
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
         if w_array is None:
-            if (    not space.isinstance_w(w_object, space.w_bytes) and 
+            if (    not space.isinstance_w(w_object, space.w_bytes) and
                     not space.isinstance_w(w_object, space.w_unicode) and
                     not isinstance(w_object, W_GenericBox)):
                 # use buffer interface
@@ -551,7 +551,7 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.call_method(w_buffer, '__buffer__', 
+        w_buffer = space.call_method(w_buffer, '__buffer__',
                                     space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -807,6 +807,15 @@
     def buffer_w(self, space, flags):
         return self.implementation.get_buffer(space, flags)
 
+    def readbuf_w(self, space):
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_readbuf()
+
+    def writebuf_w(self, space):
+        return self.implementation.get_buffer(space, space.BUF_FULL).as_writebuf()
+
+    def charbuf_w(self, space):
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
+
     def descr_get_data(self, space):
         return space.newbuffer(
             self.implementation.get_buffer(space, space.BUF_FULL).as_writebuf())
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -454,8 +454,8 @@
             return Float64(self.space).box(self.unbox(v))
         # numpy 1.10 compatibility
         raise oefmt(self.space.w_TypeError, "ufunc casting failure")
-            
-            
+
+
 
 class Integer(Primitive):
     _mixin_ = True

From pypy.commits at gmail.com  Thu May 11 11:58:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 May 2017 08:58:09 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Restrict marshalling to
 old-style buffers
Message-ID: <59148a11.e9ae500a.d202.192c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91254:647b63731eca
Date: 2017-05-11 16:53 +0100
http://bitbucket.org/pypy/pypy/changeset/647b63731eca/

Log:	Restrict marshalling to old-style buffers

diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -7,6 +7,7 @@
 from pypy.interpreter.special import Ellipsis
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter import unicodehelper
+from pypy.interpreter.buffer import BufferInterfaceNotFound
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
@@ -73,14 +74,12 @@
                 func(space, w_obj, m)
                 return
 
-    # any unknown object implementing the buffer protocol is
+    # any unknown object implementing the old-style buffer protocol is
     # accepted and encoded as a plain string
     try:
-        s = space.readbuf_w(w_obj)
-    except OperationError as e:
-        if e.match(space, space.w_TypeError):
-            raise oefmt(space.w_ValueError, "unmarshallable object")
-        raise
+        s = w_obj.readbuf_w(space)
+    except BufferInterfaceNotFound:
+        raise oefmt(space.w_ValueError, "unmarshallable object")
     m.atom_str(TYPE_STRING, s.as_str())
 
 def get_unmarshallers():

From pypy.commits at gmail.com  Thu May 11 11:58:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 11 May 2017 08:58:11 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Don't bother preventing
 _numpypy.frombuffer() from working on memoryviews
Message-ID: <59148a13.edb7500a.880d7.249e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91255:89ffc4931b4b
Date: 2017-05-11 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/89ffc4931b4b/

Log:	Don't bother preventing _numpypy.frombuffer() from working on
	memoryviews

diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3611,8 +3611,6 @@
         import numpy as np
         exc = raises(AttributeError, np.frombuffer, None)
         assert str(exc.value) == "'NoneType' object has no attribute '__buffer__'"
-        exc = raises(AttributeError, np.frombuffer, memoryview(self.data))
-        assert str(exc.value) == "'memoryview' object has no attribute '__buffer__'"
         exc = raises(ValueError, np.frombuffer, self.data, 'S0')
         assert str(exc.value) == "itemsize cannot be zero in type"
         exc = raises(ValueError, np.frombuffer, self.data, offset=-1)
@@ -3684,7 +3682,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
         dt1 = np.dtype(
              [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')],
@@ -3695,7 +3693,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
     def test_fromstring(self):
         import sys

From pypy.commits at gmail.com  Thu May 11 12:21:35 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 11 May 2017 09:21:35 -0700 (PDT)
Subject: [pypy-commit] pypy str-measure: improve the wrapper
Message-ID: <59148f8f.2db3500a.52435.4a65@mx.google.com>

Author: fijal
Branch: str-measure
Changeset: r91256:92c947349352
Date: 2017-05-11 18:20 +0200
http://bitbucket.org/pypy/pypy/changeset/92c947349352/

Log:	improve the wrapper

diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -96,6 +96,12 @@
         return space.text_w(space.str(self))
 
     def unicode_w(self, space):
+        from pypy.module.__pypy__.interp_debug import get_str_debug_file
+        w_file = get_str_debug_file(space)
+        if w_file is not None:
+            space.call_function(space.getattr(w_file, space.newtext("write")),
+                space.newtext("unicode_w %d\n" % (self._string_id,)))
+        
         return self._value
 
     def readbuf_w(self, space):
@@ -986,8 +992,18 @@
 def setup():
     from pypy.module.__pypy__.interp_debug import get_str_debug_file
 
+    def lines_for_arg(argname, func):
+        return [
+            "        if isinstance(%s, W_UnicodeObject):" % argname,
+            "            txt = '%s ' + str(self._string_id) + ' ' + str(%s._string_id) + '\\n'" % (func.func_name, argname),
+            "        else:",
+            "            txt = '%s ' + str(self._string_id) + ' string\\n'" % func.func_name
+            ]
+
+
     def wrap(func):
-        d = {'orig': func, 'get_str_debug_file': get_str_debug_file}
+        d = {'orig': func, 'get_str_debug_file': get_str_debug_file,
+        'W_UnicodeObject': W_UnicodeObject}
         name = func.__name__
         orig_args = list(func.__code__.co_varnames[:func.__code__.co_argcount])
         args = orig_args[:]
@@ -999,8 +1015,20 @@
         func_args = ", ".join(args)
         lines = ["def %s(%s):" % (name, func_args),
         "    w_file = get_str_debug_file(space)",
-        "    if w_file is not None:",
-        "        txt = '%s ' + str(self._string_id) + '\\n'" % func.func_name,
+        "    if w_file is not None:"]
+        if name in ['descr_eq', 'descr_add', 'descr_ne', 'descr_lt', 'descr_gt', 'descr_ge', 'descr_le']:
+            lines += lines_for_arg('w_other', func)
+        elif name == 'descr_startswith':
+            lines += lines_for_arg('w_prefix', func)
+        elif name == 'descr_endswith':
+            lines += lines_for_arg('w_suffix', func)
+        elif name in ['descr_find', 'descr_rfind', 'descr_index', 'descr_count']:
+            lines += lines_for_arg('w_sub', func)
+        else:
+            lines += [
+            "        txt = '%s ' + str(self._string_id) + '\\n'" % func.func_name,
+            ]
+        lines += [
         "        space.call_function(space.getattr(w_file, space.newtext('write')), space.newtext(txt))",
         "    return orig(%s)" % (", ".join(orig_args),)]
         exec "\n".join(lines) in d

From pypy.commits at gmail.com  Thu May 11 17:14:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 11 May 2017 14:14:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Disable an assert that fails also on
 CPython, where failure is better just ignored at least on PyPy
Message-ID: <5914d41f.bb87500a.1f2df.2a9b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91257:e300fd927c59
Date: 2017-05-11 23:13 +0200
http://bitbucket.org/pypy/pypy/changeset/e300fd927c59/

Log:	Disable an assert that fails also on CPython, where failure is
	better just ignored at least on PyPy

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -1004,7 +1004,8 @@
                 w_newkey = w_newvalue
             else:
                 w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
-            assert groupby.w_currvalue is None
+            #assert groupby.w_currvalue is None
+            # ^^^ check disabled, see http://bugs.python.org/issue30347
             groupby.w_currkey = w_newkey
             groupby.w_currvalue = w_newvalue
 

From pypy.commits at gmail.com  Fri May 12 10:10:52 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 12 May 2017 07:10:52 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Use double
 instead of float
Message-ID: <5915c26c.6c8edf0a.c2d64.a8d2@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2059:daf9d599a698
Date: 2017-05-12 14:14 +0200
http://bitbucket.org/pypy/stmgc/changeset/daf9d599a698/

Log:	Use double instead of float

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -5,6 +5,7 @@
 
 #include "finalizer.h"
 #include <math.h>
+#include <inttypes.h>
 
 /************************************************************/
 
@@ -23,12 +24,12 @@
 uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
 // uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
 
-#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001f)
+#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001)
 
-static float get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
+static double get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
     const int multiplier = 100;
-    float previous = tl->relative_transaction_length;
-    float new = previous;
+    double previous = tl->relative_transaction_length;
+    double new = previous;
     if (aborts) {
         tl->transaction_length_backoff = 3;
         if (previous > STM_MIN_RELATIVE_TRANSACTION_LENGTH) {
@@ -49,16 +50,16 @@
 }
 
 static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) {
-    if (!tl->initialized) {
+    if (!tl->initialized) { // TODO move to setup.c
         tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
         tl->initialized = true;
     }
-    float new = get_new_transaction_length(tl, aborts);
+    double new = get_new_transaction_length(tl, aborts);
     tl->relative_transaction_length = new;
 }
 
 static uintptr_t stm_get_transaction_length(stm_thread_local_t *tl) {
-    float relative_additional_length = tl->relative_transaction_length;
+    double relative_additional_length = tl->relative_transaction_length;
     if (timing_enabled()) {
         struct timespec relative_length = {
             .tv_sec = (int)relative_additional_length,
@@ -70,8 +71,10 @@
             STM_SINGLE_THREAD_MODE_ADAPTIVE,
             &stm_duration_payload);
     }
-    return DEFAULT_FILL_MARK_NURSERY_BYTES +
+    uintptr_t result = DEFAULT_FILL_MARK_NURSERY_BYTES +
         (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
+    printf("%020" PRIxPTR "\n", result);
+    return result;
 }
 
 
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -90,7 +90,7 @@
     intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
     /* adaptive single thread mode */
-    float relative_transaction_length;
+    double relative_transaction_length;
     int transaction_length_backoff;
     bool initialized;
 } stm_thread_local_t;

From pypy.commits at gmail.com  Fri May 12 10:10:54 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 12 May 2017 07:10:54 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Fix nested
 measurement of waiting time and time in validation; 
Message-ID: <5915c26e.6896df0a.11305.9237@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2060:66afe82c56ce
Date: 2017-05-12 16:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/66afe82c56ce/

Log:	Fix nested measurement of waiting time and time in validation; fix
	transaction size was not exponentially reduced in case of a retry

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -464,7 +464,9 @@
 #endif
 
     if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-        wait_for_inevitable();
+        pause_timer();
+        wait_for_inevitable(); // TODO may abort!! timing event lost
+        continue_timer();
         goto retry_from_start;   /* redo _stm_validate() now */
     }
 
@@ -1165,11 +1167,10 @@
     if (number_of_segments_in_use() < 2) {
         stm_become_inevitable(tl, "single thread mode");
     }
-    if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
-                                 in abort_data_structures_from_segment_num() */
-        STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
+    /* TODO remove: else, 'nursery_mark' was already set
+        in abort_data_structures_from_segment_num() */
+    STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
                                         stm_get_transaction_length(tl));
-    }
     return repeat_count;
 }
 

From pypy.commits at gmail.com  Fri May 12 10:29:09 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:09 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: start to implement
 llop.gc_store_indexed;
 still missing implementation in the C backend and the JIT
Message-ID: <5915c6b5.0486df0a.450ac.04a1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91258:4be2157b169f
Date: 2017-05-11 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/4be2157b169f/

Log:	WIP: start to implement llop.gc_store_indexed; still missing
	implementation in the C backend and the JIT

diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -397,6 +397,7 @@
     'raw_store':            LLOp(canrun=True),
     'bare_raw_store':       LLOp(),
     'gc_load_indexed':      LLOp(sideeffects=False, canrun=True),
+    'gc_store_indexed':     LLOp(canrun=True),
     'track_alloc_start':    LLOp(),
     'track_alloc_stop':     LLOp(),
     'adr_add':              LLOp(canfold=True),
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -728,6 +728,17 @@
     return p[0]
 op_gc_load_indexed.need_result_type = True
 
+def op_gc_store_indexed(p, index, scale, base_ofs, newvalue):
+    # 'base_ofs' should be a CompositeOffset(..., ArrayItemsOffset).
+    # 'scale' should be a llmemory.sizeof().
+    from rpython.rtyper.lltypesystem import rffi
+    TVAL = lltype.typeOf(newvalue)
+    ofs = base_ofs + scale * index
+    if isinstance(ofs, int):
+        return op_raw_store(p, ofs, newvalue)
+    p = rffi.cast(rffi.CArrayPtr(TVAL), llmemory.cast_ptr_to_adr(p) + ofs)
+    p[0] = newvalue
+
 def op_likely(x):
     assert isinstance(x, bool)
     return x
diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -5,6 +5,9 @@
 from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rlib.rarithmetic import r_singlefloat
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+                              ll_for_resizable_list)
+
 
 def str_gc_load(TYPE, buf, offset):
     base_ofs = (llmemory.offsetof(STR, 'chars') +
@@ -14,6 +17,19 @@
     return llop.gc_load_indexed(TYPE, lls, offset,
                                 scale_factor, base_ofs)
 
+
+def list_gc_store(TYPE, lst, offset, value):
+    value = lltype.cast_primitive(TYPE, value)
+    ll_data = ll_for_resizable_list(lst)
+    ll_items = ll_data.items
+    LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
+    base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
+    scale_factor = llmemory.sizeof(lltype.Char)
+    return llop.gc_store_indexed(lltype.Void, ll_items, offset,
+                                 scale_factor, base_ofs, value)
+
+
+
 class BaseLLOpTest(object):
     
     def test_gc_load_indexed(self):
@@ -27,6 +43,14 @@
         val = self.gc_load_from_string(rffi.INT, buf, 12)
         assert val == 0x12345678
 
+    def test_gc_store_indexed(self):
+        expected = struct.pack('i', 0x12345678)
+        n = len(expected)
+        lst = resizable_list_supporting_raw_ptr(['\x00']*n)
+        list_gc_store(rffi.INT, lst, 0, 0x12345678)
+        buf = ''.join(lst)
+        assert buf == expected
+
 
 class TestDirect(BaseLLOpTest):
 

From pypy.commits at gmail.com  Fri May 12 10:29:11 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:11 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: bah,
 actually TEST the rtyping of gc_store_index, and fix it
Message-ID: <5915c6b7.5987df0a.8d41c.4a23@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91259:1520c1ffb68f
Date: 2017-05-11 18:41 +0200
http://bitbucket.org/pypy/pypy/changeset/1520c1ffb68f/

Log:	bah, actually TEST the rtyping of gc_store_index, and fix it

diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -18,15 +18,18 @@
                                 scale_factor, base_ofs)
 
 
-def list_gc_store(TYPE, lst, offset, value):
-    value = lltype.cast_primitive(TYPE, value)
+def newlist_and_gc_store(TYPE, value):
+    size = rffi.sizeof(TYPE)
+    lst = resizable_list_supporting_raw_ptr(['\x00']*size)
     ll_data = ll_for_resizable_list(lst)
     ll_items = ll_data.items
     LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
     base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
     scale_factor = llmemory.sizeof(lltype.Char)
-    return llop.gc_store_indexed(lltype.Void, ll_items, offset,
-                                 scale_factor, base_ofs, value)
+    value = lltype.cast_primitive(TYPE, value)
+    llop.gc_store_indexed(lltype.Void, ll_items, 0,
+                          scale_factor, base_ofs, value)
+    return lst
 
 
 
@@ -45,9 +48,7 @@
 
     def test_gc_store_indexed(self):
         expected = struct.pack('i', 0x12345678)
-        n = len(expected)
-        lst = resizable_list_supporting_raw_ptr(['\x00']*n)
-        list_gc_store(rffi.INT, lst, 0, 0x12345678)
+        lst = self.newlist_and_gc_store(rffi.INT, 0x12345678)
         buf = ''.join(lst)
         assert buf == expected
 
@@ -57,9 +58,18 @@
     def gc_load_from_string(self, TYPE, buf, offset):
         return str_gc_load(TYPE, buf, offset)
 
+    def newlist_and_gc_store(self, TYPE, value):
+        return newlist_and_gc_store(TYPE, value)
+
 class TestRTyping(BaseLLOpTest, BaseRtypingTest):
 
     def gc_load_from_string(self, TYPE, buf, offset):
         def fn(offset):
             return str_gc_load(TYPE, buf, offset)
         return self.interpret(fn, [offset])
+
+    def newlist_and_gc_store(self, TYPE, value):
+        def fn(value):
+            return newlist_and_gc_store(TYPE, value)
+        ll_res = self.interpret(fn, [value])
+        return list(ll_res.items)

From pypy.commits at gmail.com  Fri May 12 10:29:17 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:17 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: hoorray! Implement the last
 bits of gc_store_indexed in llgraph and finally the test passes :)
Message-ID: <5915c6bd.c7a1df0a.23d0.25f2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91262:0b70e69aebec
Date: 2017-05-12 00:59 +0200
http://bitbucket.org/pypy/pypy/changeset/0b70e69aebec/

Log:	hoorray! Implement the last bits of gc_store_indexed in llgraph and
	finally the test passes :)

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -747,11 +747,21 @@
         return llop.gc_load_indexed(longlong.FLOATSTORAGE,
                                     struct, index, scale, base_ofs)
 
-    def bh_gc_store_indexed_i(self, struct, index, scale, base_ofs, val, bytes):
+    def bh_gc_store_indexed_i(self, struct, index, scale, base_ofs, val, bytes,
+                              descr):
         T = self._get_int_type_from_size(bytes)
         val = lltype.cast_primitive(T, val)
         llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
 
+    def bh_gc_store_indexed(self, struct, index, scale, base_ofs, val, bytes,
+                            descr):
+        if descr.A.OF == lltype.Float:
+            XXX
+            self.bh_raw_store_f(struct, offset, newvalue, descr)
+        else:
+            self.bh_gc_store_indexed_i(struct, index, scale, base_ofs,
+                                       val, bytes, descr)
+
     def bh_increment_debug_counter(self, addr):
         p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
         p[0] += 1
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -264,7 +264,8 @@
         import pdb;pdb.set_trace()
     else:
         intval = valuebox.getint()
-        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, intval, bytes)
+        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, intval, bytes,
+                                  arraydescr)
 
 
 def exec_new_with_vtable(cpu, descr):
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
--- a/rpython/jit/metainterp/test/test_llop.py
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -29,9 +29,12 @@
             return longlong.int2singlefloat(res)
         return res
 
-    def newlist_and_gc_store(self, TYPE, value):
+    def newlist_and_gc_store(self, TYPE, value, expected):
         def f(value):
-            return newlist_and_gc_store(TYPE, value)
+            lst = newlist_and_gc_store(TYPE, value)
+            got = ''.join(lst)
+            assert got == expected
+            return len(got)
         return self.interp_operations(f, [value], supports_singlefloats=True)
 
 
diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -48,9 +48,7 @@
 
     def test_gc_store_indexed(self):
         expected = struct.pack('i', 0x12345678)
-        lst = self.newlist_and_gc_store(rffi.INT, 0x12345678)
-        buf = ''.join(lst)
-        assert buf == expected
+        self.newlist_and_gc_store(rffi.INT, 0x12345678, expected)
 
 
 class TestDirect(BaseLLOpTest):
@@ -58,8 +56,10 @@
     def gc_load_from_string(self, TYPE, buf, offset):
         return str_gc_load(TYPE, buf, offset)
 
-    def newlist_and_gc_store(self, TYPE, value):
-        return newlist_and_gc_store(TYPE, value)
+    def newlist_and_gc_store(self, TYPE, value, expected):
+        got = newlist_and_gc_store(TYPE, value)
+        got = ''.join(got)
+        assert got == expected
 
 class TestRTyping(BaseLLOpTest, BaseRtypingTest):
 
@@ -68,8 +68,9 @@
             return str_gc_load(TYPE, buf, offset)
         return self.interpret(fn, [offset])
 
-    def newlist_and_gc_store(self, TYPE, value):
+    def newlist_and_gc_store(self, TYPE, value, expected):
         def fn(value):
             return newlist_and_gc_store(TYPE, value)
         ll_res = self.interpret(fn, [value])
-        return list(ll_res.items)
+        got = ''.join(ll_res.items)
+        assert got == expected
diff --git a/rpython/translator/c/test/test_llop.py b/rpython/translator/c/test/test_llop.py
--- a/rpython/translator/c/test/test_llop.py
+++ b/rpython/translator/c/test/test_llop.py
@@ -26,7 +26,7 @@
         x = fn(buf, offset)
         return lltype.cast_primitive(TYPE, x)
 
-    def newlist_and_gc_store(self, TYPE, value):
+    def newlist_and_gc_store(self, TYPE, value, expected):
         if TYPE not in self.cache:
             assert isinstance(TYPE, lltype.Primitive)
             if TYPE in (lltype.Float, lltype.SingleFloat):
@@ -43,4 +43,5 @@
             self.cache[TYPE] = fn
         #
         fn = self.cache[TYPE]
-        return fn(value)
+        got = fn(value)
+        assert got == expected

From pypy.commits at gmail.com  Fri May 12 10:29:13 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:13 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: test and implement
 gc_store_indexed in the C backend
Message-ID: <5915c6b9.0798df0a.c475d.b580@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91260:884703561c51
Date: 2017-05-11 18:50 +0200
http://bitbucket.org/pypy/pypy/changeset/884703561c51/

Log:	test and implement gc_store_indexed in the C backend

diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -311,7 +311,8 @@
     def gen_op(self, op):
         macro = 'OP_%s' % op.opname.upper()
         line = None
-        if op.opname.startswith('gc_') and op.opname != 'gc_load_indexed':
+        if (op.opname.startswith('gc_') and
+            op.opname not in ('gc_load_indexed', 'gc_store_indexed')):
             meth = getattr(self.gcpolicy, macro, None)
             if meth:
                 line = meth(self, op)
@@ -727,6 +728,19 @@
           "%(base_ofs)s + %(scale)s * %(index)s))[0];"
           % locals())
 
+    def OP_GC_STORE_INDEXED(self, op):
+        addr = self.expr(op.args[0])
+        index = self.expr(op.args[1])
+        scale = self.expr(op.args[2])
+        base_ofs = self.expr(op.args[3])
+        value = self.expr(op.args[4])
+        TYPE = op.args[4].concretetype
+        typename = cdecl(self.db.gettype(TYPE).replace('@', '*@'), '')
+        return (
+          "((%(typename)s) (((char *)%(addr)s) + "
+          "%(base_ofs)s + %(scale)s * %(index)s))[0] = %(value)s;"
+          % locals())
+
     def OP_CAST_PRIMITIVE(self, op):
         TYPE = self.lltypemap(op.result)
         val =  self.expr(op.args[0])
diff --git a/rpython/translator/c/test/test_llop.py b/rpython/translator/c/test/test_llop.py
--- a/rpython/translator/c/test/test_llop.py
+++ b/rpython/translator/c/test/test_llop.py
@@ -1,5 +1,6 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi
-from rpython.rtyper.test.test_llop import BaseLLOpTest, str_gc_load
+from rpython.rtyper.test.test_llop import (BaseLLOpTest, str_gc_load,
+                                           newlist_and_gc_store)
 from rpython.translator.c.test.test_genc import compile
 
 
@@ -24,3 +25,22 @@
         fn = self.cache[TYPE]
         x = fn(buf, offset)
         return lltype.cast_primitive(TYPE, x)
+
+    def newlist_and_gc_store(self, TYPE, value):
+        if TYPE not in self.cache:
+            assert isinstance(TYPE, lltype.Primitive)
+            if TYPE in (lltype.Float, lltype.SingleFloat):
+                argtype = float
+            else:
+                argtype = int
+
+            def llf(value):
+                value = lltype.cast_primitive(TYPE, value)
+                lst = newlist_and_gc_store(TYPE, value)
+                return ''.join(lst)
+
+            fn = compile(llf, [argtype])
+            self.cache[TYPE] = fn
+        #
+        fn = self.cache[TYPE]
+        return fn(value)

From pypy.commits at gmail.com  Fri May 12 10:29:19 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:19 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add JIT support for
 gc_store_indexed of floats
Message-ID: <5915c6bf.14ae1c0a.4d1fb.e213@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91263:84c40c5d2545
Date: 2017-05-12 01:14 +0200
http://bitbucket.org/pypy/pypy/changeset/84c40c5d2545/

Log:	add JIT support for gc_store_indexed of floats

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -753,11 +753,18 @@
         val = lltype.cast_primitive(T, val)
         llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
 
+    def bh_gc_store_indexed_f(self, struct, index, scale, base_ofs, val, bytes,
+                              descr):
+        if bytes != 8:
+            raise Exception("gc_store_indexed_f is only for 'double'!")
+        val = longlong.getrealfloat(val)
+        llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
+
     def bh_gc_store_indexed(self, struct, index, scale, base_ofs, val, bytes,
                             descr):
         if descr.A.OF == lltype.Float:
-            XXX
-            self.bh_raw_store_f(struct, offset, newvalue, descr)
+            self.bh_gc_store_indexed_f(struct, index, scale, base_ofs,
+                                       val, bytes, descr)
         else:
             self.bh_gc_store_indexed_i(struct, index, scale, base_ofs,
                                        val, bytes, descr)
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -261,7 +261,9 @@
     if arraydescr.is_array_of_pointers():
         raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
     elif arraydescr.is_array_of_floats():
-        import pdb;pdb.set_trace()
+        floatval = valuebox.getfloat()
+        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, floatval, bytes,
+                                  arraydescr)
     else:
         intval = valuebox.getint()
         cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, intval, bytes,
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -828,9 +828,9 @@
                             self._remove_symbolics(baseofsbox), bytesbox)
 
     @arguments("box", "box", "box", "box", "box", "box", "descr")
-    def opimpl_gc_store_indexed_i(self, addrbox, indexbox,
-                                  scalebox, baseofsbox, valuebox, bytesbox,
-                                  arraydescr):
+    def _opimpl_gc_store_indexed(self, addrbox, indexbox,
+                                 scalebox, baseofsbox, valuebox, bytesbox,
+                                 arraydescr):
         return self.execute_with_descr(rop.GC_STORE_INDEXED,
                                        arraydescr,
                                        addrbox,
@@ -839,6 +839,8 @@
                                        self._remove_symbolics(baseofsbox),
                                        valuebox,
                                        bytesbox)
+    opimpl_gc_store_indexed_i = _opimpl_gc_store_indexed
+    opimpl_gc_store_indexed_f = _opimpl_gc_store_indexed
 
     @arguments("box")
     def opimpl_hint_force_virtualizable(self, box):
diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -46,10 +46,14 @@
         val = self.gc_load_from_string(rffi.INT, buf, 12)
         assert val == 0x12345678
 
-    def test_gc_store_indexed(self):
+    def test_gc_store_indexed_int(self):
         expected = struct.pack('i', 0x12345678)
         self.newlist_and_gc_store(rffi.INT, 0x12345678, expected)
 
+    def test_gc_store_indexed_double(self):
+        expected = struct.pack('d', 123.456)
+        self.newlist_and_gc_store(rffi.DOUBLE, 123.456, expected)
+
 
 class TestDirect(BaseLLOpTest):
 

From pypy.commits at gmail.com  Fri May 12 10:29:15 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:15 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: start to add support for
 llop.gc_store_indexed in the JIT,
 which means to add stuff a bit everywhere. But interp_operations does not
 support returning GCREFs, so we need to tweak the tests differently
Message-ID: <5915c6bb.4186df0a.50a7d.a958@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91261:0f9bab52cf32
Date: 2017-05-12 00:52 +0200
http://bitbucket.org/pypy/pypy/changeset/0f9bab52cf32/

Log:	WIP: start to add support for llop.gc_store_indexed in the JIT,
	which means to add stuff a bit everywhere. But interp_operations
	does not support returning GCREFs, so we need to tweak the tests
	differently

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -716,16 +716,28 @@
         else:
             return self.bh_raw_load_i(struct, offset, descr)
 
+    def _get_int_type_from_size(self, size):
+        if   size == 1:
+            return rffi.UCHAR
+        elif size == 2:
+            return rffi.USHORT
+        elif size == 4:
+            return rffi.UINT
+        elif size == 8:
+            return rffi.ULONGLONG
+        elif size == -1:
+            return rffi.SIGNEDCHAR
+        elif size == -2:
+            return rffi.SHORT
+        elif size == -4:
+            return rffi.INT
+        elif size == -8:
+            return rffi.LONGLONG
+        else:
+            raise NotImplementedError(size)
+    
     def bh_gc_load_indexed_i(self, struct, index, scale, base_ofs, bytes):
-        if   bytes == 1: T = rffi.UCHAR
-        elif bytes == 2: T = rffi.USHORT
-        elif bytes == 4: T = rffi.UINT
-        elif bytes == 8: T = rffi.ULONGLONG
-        elif bytes == -1: T = rffi.SIGNEDCHAR
-        elif bytes == -2: T = rffi.SHORT
-        elif bytes == -4: T = rffi.INT
-        elif bytes == -8: T = rffi.LONGLONG
-        else: raise NotImplementedError(bytes)
+        T = self._get_int_type_from_size(bytes)
         x = llop.gc_load_indexed(T, struct, index, scale, base_ofs)
         return lltype.cast_primitive(lltype.Signed, x)
 
@@ -735,6 +747,11 @@
         return llop.gc_load_indexed(longlong.FLOATSTORAGE,
                                     struct, index, scale, base_ofs)
 
+    def bh_gc_store_indexed_i(self, struct, index, scale, base_ofs, val, bytes):
+        T = self._get_int_type_from_size(bytes)
+        val = lltype.cast_primitive(T, val)
+        llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
+
     def bh_increment_debug_counter(self, addr):
         p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
         p[0] += 1
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1131,6 +1131,27 @@
                               [op.args[0], op.args[1],
                                op.args[2], op.args[3], c_bytes], op.result)
 
+    def rewrite_op_gc_store_indexed(self, op):
+        T = op.args[4].concretetype
+        kind = getkind(T)[0]
+        assert kind != 'r'
+        descr = self.cpu.arraydescrof(rffi.CArray(T))
+        if (not isinstance(op.args[2], Constant) or
+            not isinstance(op.args[3], Constant)):
+            raise NotImplementedError("gc_store_indexed: 'scale' and 'base_ofs'"
+                                      " should be constants")
+        # xxx hard-code the size in bytes at translation time, which is
+        # probably fine and avoids lots of issues later
+        bytes = descr.get_item_size_in_bytes()
+        if descr.is_item_signed():
+            bytes = -bytes
+        c_bytes = Constant(bytes, lltype.Signed)
+        return SpaceOperation('gc_store_indexed_%s' % kind,
+                              [op.args[0], op.args[1], op.args[2],
+                               op.args[3], op.args[4], c_bytes, descr], None)
+
+
+
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
         if isinstance(arg0, Constant) and not arg0.value:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1478,6 +1478,13 @@
     def bhimpl_gc_load_indexed_f(cpu, addr, index, scale, base_ofs, bytes):
         return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
 
+    @arguments("cpu", "r", "i", "i", "i", "i", "i")
+    def bhimpl_gc_store_indexed_i(cpu, addr, index, scale, base_ofs, val, bytes):
+        return cpu.bh_gc_store_indexed_i(addr, index,scale,base_ofs, val, bytes)
+    @arguments("cpu", "r", "i", "i", "i", "f", "i")
+    def bhimpl_gc_store_indexed_f(cpu, addr, index, scale, base_ofs, val, bytes):
+        return cpu.bh_gc_store_indexed_f(addr, index,scale,base_ofs, val, bytes)
+
     @arguments("r", "d", "d")
     def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
         pass
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -251,6 +251,22 @@
     else:
         return BoxInt(cpu.bh_raw_load_i(addr, offset, arraydescr))
 
+def do_gc_store_indexed(cpu, _, addrbox, indexbox, scalebox,
+                        base_ofsbox, valuebox, bytesbox, arraydescr):
+    addr = addrbox.getref_base()
+    index = indexbox.getint()
+    scale = scalebox.getint()
+    base_ofs = base_ofsbox.getint()
+    bytes = bytesbox.getint()
+    if arraydescr.is_array_of_pointers():
+        raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
+    elif arraydescr.is_array_of_floats():
+        import pdb;pdb.set_trace()
+    else:
+        intval = valuebox.getint()
+        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, intval, bytes)
+
+
 def exec_new_with_vtable(cpu, descr):
     return cpu.bh_new_with_vtable(descr)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -827,6 +827,19 @@
                             self._remove_symbolics(scalebox),
                             self._remove_symbolics(baseofsbox), bytesbox)
 
+    @arguments("box", "box", "box", "box", "box", "box", "descr")
+    def opimpl_gc_store_indexed_i(self, addrbox, indexbox,
+                                  scalebox, baseofsbox, valuebox, bytesbox,
+                                  arraydescr):
+        return self.execute_with_descr(rop.GC_STORE_INDEXED,
+                                       arraydescr,
+                                       addrbox,
+                                       indexbox,
+                                       self._remove_symbolics(scalebox),
+                                       self._remove_symbolics(baseofsbox),
+                                       valuebox,
+                                       bytesbox)
+
     @arguments("box")
     def opimpl_hint_force_virtualizable(self, box):
         self.metainterp.gen_store_back_in_vable(box)
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
--- a/rpython/jit/metainterp/test/test_llop.py
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -2,7 +2,8 @@
 import sys
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.test.test_llop import BaseLLOpTest, str_gc_load
+from rpython.rtyper.test.test_llop import (BaseLLOpTest, str_gc_load,
+                                           newlist_and_gc_store)
 from rpython.jit.codewriter import longlong
 from rpython.jit.metainterp.history import getkind
 from rpython.jit.metainterp.test.support import LLJitMixin
@@ -28,6 +29,12 @@
             return longlong.int2singlefloat(res)
         return res
 
+    def newlist_and_gc_store(self, TYPE, value):
+        def f(value):
+            return newlist_and_gc_store(TYPE, value)
+        return self.interp_operations(f, [value], supports_singlefloats=True)
+
+
     def test_force_virtual_str_storage(self):
         byteorder = sys.byteorder
         size = rffi.sizeof(lltype.Signed)

From pypy.commits at gmail.com  Fri May 12 10:29:25 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:25 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement support for
 gc_store_indexed also in llsupport: this fixes the tests for the x86
 backend, and hopefully for the other backends as well
Message-ID: <5915c6c5.229adf0a.bdbad.8cc0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91266:88ae2f6e9df5
Date: 2017-05-12 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/88ae2f6e9df5/

Log:	implement support for gc_store_indexed also in llsupport: this fixes
	the tests for the x86 backend, and hopefully for the other backends
	as well

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -754,6 +754,16 @@
         offset = base_ofs + scale * index
         return self.read_float_at_mem(addr, offset)
 
+    def bh_gc_store_indexed_i(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_int_at_mem(addr, offset, bytes, val)
+
+    def bh_gc_store_indexed_f(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_float_at_mem(addr, offset, val)
+
     def bh_new(self, sizedescr):
         return self.gc_ll_descr.gc_malloc(sizedescr)
 
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -262,7 +262,7 @@
         raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
     elif arraydescr.is_array_of_floats():
         floatval = valuebox.getfloat()
-        cpu.bh_gc_store_indexed_i(addr, index, floatval, scale, base_ofs, bytes,
+        cpu.bh_gc_store_indexed_f(addr, index, floatval, scale, base_ofs, bytes,
                                   arraydescr)
     else:
         intval = valuebox.getint()
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
--- a/rpython/jit/metainterp/test/test_llop.py
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -33,7 +33,9 @@
         def f(value):
             lst = newlist_and_gc_store(TYPE, value)
             got = ''.join(lst)
-            assert got == expected
+            if got != expected:
+                # I'm not sure why, but if I use an assert, the test doesn't fail
+                raise ValueError('got != expected')
             return len(got)
         return self.interp_operations(f, [value], supports_singlefloats=True)
 

From pypy.commits at gmail.com  Fri May 12 10:29:27 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:27 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add the llop test also for ARM
Message-ID: <5915c6c7.50161c0a.48204.ec2f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91267:ca663c6eea4d
Date: 2017-05-12 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/ca663c6eea4d/

Log:	add the llop test also for ARM

diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -0,0 +1,9 @@
+from rpython.jit.backend.arm.test.support import JitARMMixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(JitARMMixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+    pass
+

From pypy.commits at gmail.com  Fri May 12 10:29:21 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:21 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a passing test for single
 floats
Message-ID: <5915c6c1.46371c0a.1062d.b22a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91264:a735e006ad8a
Date: 2017-05-12 01:18 +0200
http://bitbucket.org/pypy/pypy/changeset/a735e006ad8a/

Log:	add a passing test for single floats

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -751,6 +751,8 @@
                               descr):
         T = self._get_int_type_from_size(bytes)
         val = lltype.cast_primitive(T, val)
+        if descr.A.OF == lltype.SingleFloat:
+            val = longlong.int2singlefloat(val)
         llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
 
     def bh_gc_store_indexed_f(self, struct, index, scale, base_ofs, val, bytes,
diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -54,6 +54,10 @@
         expected = struct.pack('d', 123.456)
         self.newlist_and_gc_store(rffi.DOUBLE, 123.456, expected)
 
+    def test_gc_store_indexed_float(self):
+        expected = struct.pack('f', 123.456)
+        self.newlist_and_gc_store(rffi.FLOAT, 123.456, expected)
+
 
 class TestDirect(BaseLLOpTest):
 

From pypy.commits at gmail.com  Fri May 12 10:29:23 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 07:29:23 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: shuffle the order of arguments
 of llop.gc_store_indexed to match the existing rop.GC_STORE_INDEXED
Message-ID: <5915c6c3.ddb1df0a.e4770.6bcb@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91265:d5941a454db5
Date: 2017-05-12 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/d5941a454db5/

Log:	shuffle the order of arguments of llop.gc_store_indexed to match the
	existing rop.GC_STORE_INDEXED

diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -747,29 +747,29 @@
         return llop.gc_load_indexed(longlong.FLOATSTORAGE,
                                     struct, index, scale, base_ofs)
 
-    def bh_gc_store_indexed_i(self, struct, index, scale, base_ofs, val, bytes,
+    def bh_gc_store_indexed_i(self, struct, index, val, scale, base_ofs, bytes,
                               descr):
         T = self._get_int_type_from_size(bytes)
         val = lltype.cast_primitive(T, val)
         if descr.A.OF == lltype.SingleFloat:
             val = longlong.int2singlefloat(val)
-        llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
 
-    def bh_gc_store_indexed_f(self, struct, index, scale, base_ofs, val, bytes,
+    def bh_gc_store_indexed_f(self, struct, index, val, scale, base_ofs, bytes,
                               descr):
         if bytes != 8:
             raise Exception("gc_store_indexed_f is only for 'double'!")
         val = longlong.getrealfloat(val)
-        llop.gc_store_indexed(lltype.Void, struct, index, scale, base_ofs, val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
 
-    def bh_gc_store_indexed(self, struct, index, scale, base_ofs, val, bytes,
+    def bh_gc_store_indexed(self, struct, index, val, scale, base_ofs, bytes,
                             descr):
         if descr.A.OF == lltype.Float:
-            self.bh_gc_store_indexed_f(struct, index, scale, base_ofs,
-                                       val, bytes, descr)
+            self.bh_gc_store_indexed_f(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
         else:
-            self.bh_gc_store_indexed_i(struct, index, scale, base_ofs,
-                                       val, bytes, descr)
+            self.bh_gc_store_indexed_i(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
 
     def bh_increment_debug_counter(self, addr):
         p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1132,19 +1132,18 @@
                                op.args[2], op.args[3], c_bytes], op.result)
 
     def rewrite_op_gc_store_indexed(self, op):
-        T = op.args[4].concretetype
+        T = op.args[2].concretetype
         kind = getkind(T)[0]
         assert kind != 'r'
         descr = self.cpu.arraydescrof(rffi.CArray(T))
-        if (not isinstance(op.args[2], Constant) or
-            not isinstance(op.args[3], Constant)):
+        if (not isinstance(op.args[3], Constant) or
+            not isinstance(op.args[4], Constant)):
             raise NotImplementedError("gc_store_indexed: 'scale' and 'base_ofs'"
                                       " should be constants")
-        # xxx hard-code the size in bytes at translation time, which is
-        # probably fine and avoids lots of issues later
+        # According to the comment in resoperation.py, "itemsize is not signed
+        # (always > 0)", so we don't need the "bytes = -bytes" line which is
+        # in rewrite_op_gc_load_indexed
         bytes = descr.get_item_size_in_bytes()
-        if descr.is_item_signed():
-            bytes = -bytes
         c_bytes = Constant(bytes, lltype.Signed)
         return SpaceOperation('gc_store_indexed_%s' % kind,
                               [op.args[0], op.args[1], op.args[2],
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1479,11 +1479,11 @@
         return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
 
     @arguments("cpu", "r", "i", "i", "i", "i", "i")
-    def bhimpl_gc_store_indexed_i(cpu, addr, index, scale, base_ofs, val, bytes):
-        return cpu.bh_gc_store_indexed_i(addr, index,scale,base_ofs, val, bytes)
-    @arguments("cpu", "r", "i", "i", "i", "f", "i")
-    def bhimpl_gc_store_indexed_f(cpu, addr, index, scale, base_ofs, val, bytes):
-        return cpu.bh_gc_store_indexed_f(addr, index,scale,base_ofs, val, bytes)
+    def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs, bytes):
+        return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs, bytes)
+    @arguments("cpu", "r", "i", "f", "i", "i", "i")
+    def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs, bytes):
+        return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs, bytes)
 
     @arguments("r", "d", "d")
     def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -251,8 +251,8 @@
     else:
         return BoxInt(cpu.bh_raw_load_i(addr, offset, arraydescr))
 
-def do_gc_store_indexed(cpu, _, addrbox, indexbox, scalebox,
-                        base_ofsbox, valuebox, bytesbox, arraydescr):
+def do_gc_store_indexed(cpu, _, addrbox, indexbox, valuebox, scalebox,
+                        base_ofsbox, bytesbox, arraydescr):
     addr = addrbox.getref_base()
     index = indexbox.getint()
     scale = scalebox.getint()
@@ -262,11 +262,11 @@
         raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
     elif arraydescr.is_array_of_floats():
         floatval = valuebox.getfloat()
-        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, floatval, bytes,
+        cpu.bh_gc_store_indexed_i(addr, index, floatval, scale, base_ofs, bytes,
                                   arraydescr)
     else:
         intval = valuebox.getint()
-        cpu.bh_gc_store_indexed_i(addr, index, scale, base_ofs, intval, bytes,
+        cpu.bh_gc_store_indexed_i(addr, index, intval, scale, base_ofs, bytes,
                                   arraydescr)
 
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -828,16 +828,16 @@
                             self._remove_symbolics(baseofsbox), bytesbox)
 
     @arguments("box", "box", "box", "box", "box", "box", "descr")
-    def _opimpl_gc_store_indexed(self, addrbox, indexbox,
-                                 scalebox, baseofsbox, valuebox, bytesbox,
+    def _opimpl_gc_store_indexed(self, addrbox, indexbox, valuebox,
+                                 scalebox, baseofsbox, bytesbox,
                                  arraydescr):
         return self.execute_with_descr(rop.GC_STORE_INDEXED,
                                        arraydescr,
                                        addrbox,
                                        indexbox,
+                                       valuebox,
                                        self._remove_symbolics(scalebox),
                                        self._remove_symbolics(baseofsbox),
-                                       valuebox,
                                        bytesbox)
     opimpl_gc_store_indexed_i = _opimpl_gc_store_indexed
     opimpl_gc_store_indexed_f = _opimpl_gc_store_indexed
diff --git a/rpython/rtyper/lltypesystem/opimpl.py b/rpython/rtyper/lltypesystem/opimpl.py
--- a/rpython/rtyper/lltypesystem/opimpl.py
+++ b/rpython/rtyper/lltypesystem/opimpl.py
@@ -728,7 +728,7 @@
     return p[0]
 op_gc_load_indexed.need_result_type = True
 
-def op_gc_store_indexed(p, index, scale, base_ofs, newvalue):
+def op_gc_store_indexed(p, index, newvalue, scale, base_ofs):
     # 'base_ofs' should be a CompositeOffset(..., ArrayItemsOffset).
     # 'scale' should be a llmemory.sizeof().
     from rpython.rtyper.lltypesystem import rffi
diff --git a/rpython/rtyper/test/test_llop.py b/rpython/rtyper/test/test_llop.py
--- a/rpython/rtyper/test/test_llop.py
+++ b/rpython/rtyper/test/test_llop.py
@@ -28,7 +28,7 @@
     scale_factor = llmemory.sizeof(lltype.Char)
     value = lltype.cast_primitive(TYPE, value)
     llop.gc_store_indexed(lltype.Void, ll_items, 0,
-                          scale_factor, base_ofs, value)
+                          value, scale_factor, base_ofs)
     return lst
 
 
diff --git a/rpython/translator/c/funcgen.py b/rpython/translator/c/funcgen.py
--- a/rpython/translator/c/funcgen.py
+++ b/rpython/translator/c/funcgen.py
@@ -731,10 +731,10 @@
     def OP_GC_STORE_INDEXED(self, op):
         addr = self.expr(op.args[0])
         index = self.expr(op.args[1])
-        scale = self.expr(op.args[2])
-        base_ofs = self.expr(op.args[3])
-        value = self.expr(op.args[4])
-        TYPE = op.args[4].concretetype
+        value = self.expr(op.args[2])
+        scale = self.expr(op.args[3])
+        base_ofs = self.expr(op.args[4])
+        TYPE = op.args[2].concretetype
         typename = cdecl(self.db.gettype(TYPE).replace('@', '*@'), '')
         return (
           "((%(typename)s) (((char *)%(addr)s) + "

From pypy.commits at gmail.com  Fri May 12 12:22:07 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 09:22:07 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement
 MutableStringBuffer.typed_write
Message-ID: <5915e12f.8a841c0a.33f41.07b5@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91268:77c4134d96d9
Date: 2017-05-12 17:36 +0200
http://bitbucket.org/pypy/pypy/changeset/77c4134d96d9/

Log:	implement MutableStringBuffer.typed_write

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -1,7 +1,8 @@
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
 from rpython.rtyper.annlowlevel import llstr, hlstr
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.buffer import Buffer
 
 class MutableStringBuffer(Buffer):
@@ -48,3 +49,12 @@
     def setzeros(self, index, count):
         for i in range(index, index+count):
             self.setitem(i, '\x00')
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        base_ofs = (llmemory.offsetof(STR, 'chars') +
+                    llmemory.itemoffsetof(STR.chars, 0))
+        scale_factor = llmemory.sizeof(lltype.Char)
+        value = lltype.cast_primitive(TP, value)
+        llop.gc_store_indexed(lltype.Void, self.ll_val, byte_offset, value,
+                              scale_factor, base_ofs)
diff --git a/rpython/rlib/test/test_mutbuffer.py b/rpython/rlib/test/test_mutbuffer.py
--- a/rpython/rlib/test/test_mutbuffer.py
+++ b/rpython/rlib/test/test_mutbuffer.py
@@ -1,4 +1,6 @@
 import pytest
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.mutbuffer import MutableStringBuffer
 
 class TestMutableStringBuffer(object):
@@ -28,3 +30,13 @@
         buf.setslice(0, 'ABCDEFGH')
         buf.setzeros(2, 3)
         assert buf.finish() == 'AB\x00\x00\x00FGH'
+
+    def test_typed_write(self):
+        expected = struct.pack('ifqd', 0x1234, 123.456, 0x12345678, 789.123)
+        buf = MutableStringBuffer(24)
+        buf.typed_write(rffi.INT, 0, 0x1234)
+        buf.typed_write(rffi.FLOAT, 4, 123.456)
+        buf.typed_write(rffi.LONGLONG, 8, 0x12345678)
+        buf.typed_write(rffi.DOUBLE, 16, 789.123)
+        s = buf.finish()
+        assert s == expected

From pypy.commits at gmail.com  Fri May 12 12:22:09 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 09:22:09 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: finally! Add a fastpath for
 packing ints :)
Message-ID: <5915e131.4a871c0a.6c2ec.4a11@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91269:196fb3e1e9b3
Date: 2017-05-12 18:19 +0200
http://bitbucket.org/pypy/pypy/changeset/196fb3e1e9b3/

Log:	finally! Add a fastpath for packing ints :)

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -14,9 +14,12 @@
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.buffer import StringBuffer
 from rpython.rlib import rarithmetic
-from rpython.rlib.buffer import CannotRead
+from rpython.rlib.buffer import CannotRead, CannotWrite
 from rpython.rtyper.lltypesystem import rffi
 
+USE_FASTPATH = True    # set to False by some tests
+ALLOW_SLOWPATH = True  # set to False by some tests
+
 native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
 
@@ -65,6 +68,21 @@
     fmtiter.advance(1)
     _pack_string(fmtiter, string, count-1)
 
+ at specialize.memo()
+def pack_fastpath(TYPE):
+    @specialize.argtype(0)
+    def do_pack_fastpath(fmtiter, value):
+        size = rffi.sizeof(TYPE)
+        pos = fmtiter.pos
+        if (not USE_FASTPATH or
+            fmtiter.bigendian != native_is_bigendian or
+            pos % size != 0):
+            raise CannotWrite
+        # the following might raise CannotWrite and abort the fastpath
+        fmtiter.result.typed_write(TYPE, fmtiter.pos, value)
+        fmtiter.advance(size)
+    return do_pack_fastpath
+
 def make_float_packer(size):
     def packer(fmtiter):
         fl = fmtiter.accept_float_arg()
@@ -124,6 +142,7 @@
     errormsg = "argument out of range for %d-byte%s integer format" % (size,
                                                                        plural)
     unroll_revrange_size = unrolling_iterable(range(size-1, -1, -1))
+    TYPE = get_rffi_int_type(size, signed)
 
     def pack_int(fmtiter):
         method = getattr(fmtiter, accept_method)
@@ -131,6 +150,14 @@
         if not min <= value <= max:
             raise StructError(errormsg)
         #
+        try:
+            pack_fastpath(TYPE)(fmtiter, value)
+            return
+        except CannotWrite:
+            if not ALLOW_SLOWPATH:
+                # we enter here only in some tests
+                raise ValueError("fastpath not taken :(")
+        #
         pos = fmtiter.pos + size - 1        
         if fmtiter.bigendian:
             for i in unroll_revrange_size:
@@ -148,9 +175,6 @@
 
 # ____________________________________________________________
 
-USE_FASTPATH = True    # set to False by some tests
-ALLOW_SLOWPATH = True  # set to False by some tests
-
 
 @specialize.memo()
 def unpack_fastpath(TYPE):
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -1,4 +1,5 @@
 import pytest
+from rpython.rlib.rarithmetic import r_ulonglong
 from rpython.rlib.rstruct import standardfmttable, nativefmttable
 from rpython.rlib.mutbuffer import MutableStringBuffer
 import struct
@@ -40,6 +41,23 @@
     fmt_prefix = None
     fmttable = None
 
+    USE_FASTPATH = True
+    ALLOW_SLOWPATH = True
+    
+    def setup_method(self, meth):
+        standardfmttable.USE_FASTPATH = self.USE_FASTPATH
+        standardfmttable.ALLOW_SLOWPATH = self.ALLOW_SLOWPATH
+
+    def teardown_method(self, meth):
+        standardfmttable.USE_FASTPATH = True
+        standardfmttable.ALLOW_SLOWPATH = True
+
+    def teardown_method(self, meth):
+        if not hasattr(self.fmttable, 'USE_FASTPATH'):
+            return
+        self.fmttable.USE_FASTPATH = self.orig_use_fastpath
+        self.fmttable.ALLOW_SLOWPATH = self.orig_allow_slowpath
+
     def mypack(self, fmt, value):
         size = struct.calcsize(fmt)
         fake_fmtiter = FakeFormatIter(self.bigendian, size, value)
@@ -69,7 +87,7 @@
         self.check("I", 0x81424344)
         self.check("q", 0x4142434445464748)
         self.check("q", -0x41B2B3B4B5B6B7B8)
-        self.check("Q", 0x8142434445464748)
+        self.check("Q", r_ulonglong(0x8142434445464748))
 
     def test_pack_ieee(self):
         self.check('f', 123.456)
@@ -107,12 +125,22 @@
     fmt_prefix = '<'
     fmttable = standardfmttable.standard_fmttable
 
+class TestPackLittleEndianSlowPath(TestPackLittleEndian):
+    USE_FASTPATH = False
+
 class TestPackBigEndian(BaseTestPack):
     bigendian = True
     fmt_prefix = '>'
     fmttable = standardfmttable.standard_fmttable
 
+class TestPackBigEndianSlowPath(TestPackBigEndian):
+    USE_FASTPATH = False
+
+
 class TestNative(BaseTestPack):
+    # native packing automatically use the proper endianess, so it should
+    # always take the fast path
+    ALLOW_SLOWPATH = False
     bigendian = nativefmttable.native_is_bigendian
     fmt_prefix = '@'
     fmttable = nativefmttable.native_fmttable

From pypy.commits at gmail.com  Fri May 12 14:20:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 May 2017 11:20:42 -0700 (PDT)
Subject: [pypy-commit] pypy PyBuffer-backport: Allow memoryviews in
 compile() and _codecs.unicode_internal_decode()
Message-ID: <5915fcfa.d4451c0a.b9b64.364a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: PyBuffer-backport
Changeset: r91270:5ea0ac27cf69
Date: 2017-05-12 19:18 +0100
http://bitbucket.org/pypy/pypy/changeset/5ea0ac27cf69/

Log:	Allow memoryviews in compile() and _codecs.unicode_internal_decode()

diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,5 +1,6 @@
 class AppTestCompile:
     def test_simple(self):
+        import sys
         co = compile('1+2', '?', 'eval')
         assert eval(co) == 3
         co = compile(buffer('1+2'), '?', 'eval')
@@ -8,8 +9,10 @@
         assert str(exc.value) == "compile() expected string without null bytes"
         exc = raises(TypeError, compile, unichr(0), '?', 'eval')
         assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
+
+        if '__pypy__' in sys.modules:
+            co = compile(memoryview('1+2'), '?', 'eval')
+            assert eval(co) == 3
         compile("from __future__ import with_statement", "<test>", "exec")
         raises(SyntaxError, compile, '-', '?', 'eval')
         raises(ValueError, compile, '"\\xt"', '?', 'eval')
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -292,8 +292,8 @@
             assert bytes2.decode("unicode_internal") == u"\U00010098"
         assert bytes.decode("unicode_internal") == u"a"
         assert _codecs.unicode_internal_decode(array.array('c', bytes))[0] == u"a"
-        exc = raises(TypeError, _codecs.unicode_internal_decode, memoryview(bytes))
-        assert str(exc.value) == "expected a readable buffer object"
+        if '__pypy__' in sys.modules:
+            assert _codecs.unicode_internal_decode(memoryview(bytes))[0] == u"a"
 
     def test_raw_unicode_escape(self):
         assert unicode("\u0663", "raw-unicode-escape") == u"\u0663"

From pypy.commits at gmail.com  Fri May 12 14:44:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 May 2017 11:44:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Document merged branch
Message-ID: <5916027a.e3b2df0a.b4029.38ef@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91273:13b0394c4ce9
Date: 2017-05-12 19:43 +0100
http://bitbucket.org/pypy/pypy/changeset/13b0394c4ce9/

Log:	Document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,8 @@
 .. branch: controller-refactor
 
 Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().

From pypy.commits at gmail.com  Fri May 12 16:20:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 12 May 2017 13:20:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59161925.ce1a1c0a.6510.a2b3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91274:f539c4765bcd
Date: 2017-05-12 21:20 +0100
http://bitbucket.org/pypy/pypy/changeset/f539c4765bcd/

Log:	hg merge default

diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -514,7 +514,14 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,8 @@
 .. branch: controller-refactor
 
 Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (for post-5.7.1 builds) with sha256 checksum 
-``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -128,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -236,6 +236,9 @@
     copy out32\*.lib <somewhere in LIB>
     xcopy /S include\openssl <somewhere in INCLUDE>
 
+For tests you will also need the dlls::
+    nmake -f ms\ntdll.mak install
+    copy out32dll\*.dll <somewhere in PATH>
 
 TkInter module support
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -245,18 +248,17 @@
 directory found for the release script, create the dlls, libs, headers and
 runtime by running::
 
-	svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
-	svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
-	cd tcl85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
-	nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
-	cd ..\..\tk85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+    svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+    svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+    cd tcl85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
+    nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+    cd ..\..\tk85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+    copy ..\..\tcltk\bin\* <somewhere in PATH>
+    copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+    xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
 
 The lzma compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -341,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -79,7 +79,7 @@
 
     def getname(self, space):
         try:
-            return space.unicode_w(space.getattr(self, space.newtext('__name__')))
+            return space.text_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return u'?'
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -43,6 +43,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         ec.compiler.validate_ast(ast_node)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags,
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -2,6 +2,7 @@
 class AppTestCompile:
 
     def test_simple(self):
+        import sys
         co = compile('1+2', '?', 'eval')
         assert eval(co) == 3
         co = compile(memoryview(b'1+2'), '?', 'eval')
@@ -66,7 +67,8 @@
         co1 = compile('print(1)', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -10,7 +10,7 @@
         raises(TypeError, _codecs.register, 1)
 
     def test_bigU_codecs(self):
-        u = '\U00010001\U00020002\U00030003\U00040004\U00050005'
+        u = u'\U00010001\U00020002\U00030003\U00040004\U00050005'
         for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
                          'utf-32', 'utf-32-le', 'utf-32-be',
                          'raw_unicode_escape',
@@ -18,7 +18,7 @@
             assert str(u.encode(encoding),encoding) == u
 
     def test_ucs4(self):
-        x = '\U00100000'
+        x = u'\U00100000'
         y = x.encode("raw-unicode-escape").decode("raw-unicode-escape")
         assert x == y
 
@@ -146,21 +146,21 @@
         import _codecs
         encoding = 'utf-8'
         check_partial = [
-                "\x00",
-                "\x00",
-                "\x00\xff",
-                "\x00\xff",
-                "\x00\xff\u07ff",
-                "\x00\xff\u07ff",
-                "\x00\xff\u07ff",
-                "\x00\xff\u07ff\u0800",
-                "\x00\xff\u07ff\u0800",
-                "\x00\xff\u07ff\u0800",
-                "\x00\xff\u07ff\u0800\uffff",
-                "\x00\xff\u07ff\u0800\uffff",
-                "\x00\xff\u07ff\u0800\uffff",
-                "\x00\xff\u07ff\u0800\uffff",
-                "\x00\xff\u07ff\u0800\uffff\U00010000",
+                u"\x00",
+                u"\x00",
+                u"\x00\xff",
+                u"\x00\xff",
+                u"\x00\xff\u07ff",
+                u"\x00\xff\u07ff",
+                u"\x00\xff\u07ff",
+                u"\x00\xff\u07ff\u0800",
+                u"\x00\xff\u07ff\u0800",
+                u"\x00\xff\u07ff\u0800",
+                u"\x00\xff\u07ff\u0800\uffff",
+                u"\x00\xff\u07ff\u0800\uffff",
+                u"\x00\xff\u07ff\u0800\uffff",
+                u"\x00\xff\u07ff\u0800\uffff",
+                u"\x00\xff\u07ff\u0800\uffff\U00010000",
             ]
 
         buffer = b''
@@ -177,20 +177,20 @@
         import _codecs
         encoding = 'utf-16'
         check_partial = [
-                    "", # first byte of BOM read
-                    "", # second byte of BOM read => byteorder known
-                    "",
-                    "\x00",
-                    "\x00",
-                    "\x00\xff",
-                    "\x00\xff",
-                    "\x00\xff\u0100",
-                    "\x00\xff\u0100",
-                    "\x00\xff\u0100\uffff",
-                    "\x00\xff\u0100\uffff",
-                    "\x00\xff\u0100\uffff",
-                    "\x00\xff\u0100\uffff",
-                    "\x00\xff\u0100\uffff\U00010000",
+                    u"", # first byte of BOM read
+                    u"", # second byte of BOM read => byteorder known
+                    u"",
+                    u"\x00",
+                    u"\x00",
+                    u"\x00\xff",
+                    u"\x00\xff",
+                    u"\x00\xff\u0100",
+                    u"\x00\xff\u0100",
+                    u"\x00\xff\u0100\uffff",
+                    u"\x00\xff\u0100\uffff",
+                    u"\x00\xff\u0100\uffff",
+                    u"\x00\xff\u0100\uffff",
+                    u"\x00\xff\u0100\uffff\U00010000",
                 ]
         buffer = b''
         result = ""
@@ -205,9 +205,9 @@
     def test_bug1098990_a(self):
         import codecs, io
         self.encoding = 'utf-8'
-        s1 = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
-        s2 = "offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
-        s3 = "next line.\r\n"
+        s1 = u"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy\r\n"
+        s2 = u"offending line: ladfj askldfj klasdj fskla dfzaskdj fasklfj laskd fjasklfzzzzaa%whereisthis!!!\r\n"
+        s3 = u"next line.\r\n"
 
         s = (s1+s2+s3).encode(self.encoding)
         stream = io.BytesIO(s)
@@ -215,16 +215,16 @@
         assert reader.readline() == s1
         assert reader.readline() == s2
         assert reader.readline() == s3
-        assert reader.readline() == ""
+        assert reader.readline() == u""
 
     def test_bug1098990_b(self):
         import codecs, io
         self.encoding = 'utf-8'
-        s1 = "aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
-        s2 = "bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
-        s3 = "stillokay:bbbbxx\r\n"
-        s4 = "broken!!!!badbad\r\n"
-        s5 = "againokay.\r\n"
+        s1 = u"aaaaaaaaaaaaaaaaaaaaaaaa\r\n"
+        s2 = u"bbbbbbbbbbbbbbbbbbbbbbbb\r\n"
+        s3 = u"stillokay:bbbbxx\r\n"
+        s4 = u"broken!!!!badbad\r\n"
+        s5 = u"againokay.\r\n"
 
         s = (s1+s2+s3+s4+s5).encode(self.encoding)
         stream = io.BytesIO(s)
@@ -234,7 +234,7 @@
         assert reader.readline() == s3
         assert reader.readline() == s4
         assert reader.readline() == s5
-        assert reader.readline() == ""
+        assert reader.readline() == u""
 
     def test_seek_utf16le(self):
         # all codecs should be able to encode these
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -470,10 +470,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self,
+        view = CPyBuffer(space, ptr[0], size, w_self,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return buf.wrap(space)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getwritebuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -488,10 +488,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+        view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return buf.wrap(space)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,6 +344,7 @@
 #endif
     if (m == NULL)
         INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -133,7 +133,7 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyLong_FromLong(-1);
+                    return NULL;
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
                 return PyLong_FromLong(view->len / view->itemsize);
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -34,7 +34,7 @@
 cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct)
 
 @bootstrap_function
-def init_stringobject(space):
+def init_tupleobject(space):
     "Type description of PyTupleObject"
     make_typedescr(space.w_tuple.layout.typedef,
                    basestruct=PyTupleObject.TO,
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -527,6 +527,17 @@
         assert a == []
         assert b == [(True, 9)]
 
+    def test_groupby_question_43905804(self):
+        # http://stackoverflow.com/questions/43905804/
+        import itertools
+
+        inputs = ((x > 5, x) for x in range(10))
+        (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+        a = list(a)
+        b = list(b)
+        assert a == []
+        assert b == [(True, 9)]
+
     def test_iterables(self):
         import itertools
 
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -3,6 +3,7 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.listsort import make_timsort_class
+from rpython.rlib.buffer import Buffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -701,10 +702,8 @@
     def __del__(self):
         free_raw_storage(self.storage)
 
-
-class ArrayView(BufferView):
+class ArrayData(Buffer):
     _immutable_ = True
-
     def __init__(self, impl, readonly):
         self.impl = impl
         self.readonly = readonly
@@ -725,6 +724,28 @@
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
 
+
+class ArrayView(BufferView):
+    _immutable_ = True
+
+    def __init__(self, impl, readonly):
+        self.impl = impl
+        self.readonly = readonly
+        self.data = ArrayData(impl, readonly)
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def as_readbuf(self):
+        return ArrayData(self.impl, readonly=True)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return ArrayData(self.impl, readonly=False)
+
     def getformat(self):
         sb = StringBuilder()
         self.impl.dtype.getformat(sb)
@@ -742,4 +763,5 @@
     def getstrides(self):
         return self.impl.strides
 
-
+    def get_raw_address(self):
+        return self.data.get_raw_address()
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -3,6 +3,7 @@
     WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, \
     make_weakref_descr
+from pypy.interpreter.buffer import SimpleView
 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import RAW_STORAGE_PTR
@@ -808,7 +809,8 @@
         return self.implementation.get_buffer(space, flags)
 
     def descr_get_data(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL).wrap(space)
+        return space.newmemoryview(
+            self.implementation.get_buffer(space, space.BUF_FULL))
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3602,8 +3602,6 @@
         import numpy as np
         exc = raises(AttributeError, np.frombuffer, None)
         assert str(exc.value) == "'NoneType' object has no attribute '__buffer__'"
-        exc = raises(AttributeError, np.frombuffer, memoryview(self.data))
-        assert str(exc.value) == "'memoryview' object has no attribute '__buffer__'"
         exc = raises(ValueError, np.frombuffer, self.data, 'S0')
         assert str(exc.value) == "itemsize cannot be zero in type"
         exc = raises(ValueError, np.frombuffer, self.data, offset=-1)
@@ -3675,7 +3673,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
         dt1 = np.dtype(
              [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')],
@@ -3686,7 +3684,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -454,8 +454,8 @@
             return Float64(self.space).box(self.unbox(v))
         # numpy 1.10 compatibility
         raise oefmt(self.space.w_TypeError, "ufunc casting failure")
-            
-            
+
+
 
 class Integer(Primitive):
     _mixin_ = True
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,9 +1,9 @@
 from rpython.rlib import jit
+from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.buffer import SubBuffer
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
@@ -177,16 +177,17 @@
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
-    def __init__(self, space, format):
+    format = ""
+    size = -1
+
+    def descr__new__(space, w_subtype, __args__):
+        return space.allocate_instance(W_Struct, w_subtype)
+
+    def descr__init__(self, space, w_format):
+        format = text_or_bytes_w(space, w_format)
         self.format = format
         self.size = _calcsize(space, format)
 
-    def descr__new__(space, w_subtype, w_format):
-        format = text_or_bytes_w(space, w_format)
-        self = space.allocate_instance(W_Struct, w_subtype)
-        W_Struct.__init__(self, space, format)
-        return self
-
     def descr_pack(self, space, args_w):
         return do_pack(space, jit.promote_string(self.format), args_w)
 
@@ -206,6 +207,7 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
+    __init__=interp2app(W_Struct.descr__init__),
     format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
     size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
@@ -223,8 +225,8 @@
 )
 
 def iter_unpack(space, w_format, w_buffer):
-    format = text_or_bytes_w(space, w_format)
-    w_struct = W_Struct(space, format)
+    w_struct = W_Struct()
+    w_struct.descr__init__(space, w_format)
     return W_UnpackIter(space, w_struct, w_buffer)
 
 def clearcache(space):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -455,6 +455,14 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_struct_subclass(self):
+        class S(self.struct.Struct):
+            def __init__(self):
+                assert self.size == -1
+                super(S, self).__init__('b')
+                assert self.size == 1
+        assert S().unpack(b'a') == (ord(b'a'),)
+
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -26,7 +26,6 @@
 from pypy.objspace.std.formatting import mod_format, FORMAT_BYTEARRAY
 
 
-
 class W_BytearrayObject(W_Root):
     import_from_mixin(StringMethods)
     _KIND1 = "bytearray"
@@ -1277,10 +1276,10 @@
 
 class BytearrayBuffer(Buffer):
     _immutable_ = True
-    readonly = False
 
-    def __init__(self, ba):
+    def __init__(self, ba, readonly=False):
         self.ba = ba     # the W_BytearrayObject
+        self.readonly = readonly
 
     def getlength(self):
         return self.ba._len()
diff --git a/pypy/objspace/std/intobject.py b/pypy/objspace/std/intobject.py
--- a/pypy/objspace/std/intobject.py
+++ b/pypy/objspace/std/intobject.py
@@ -768,7 +768,7 @@
     # use r_uint to perform a single comparison (this whole function is
     # getting inlined into every caller so keeping the branching to a
     # minimum is a good idea)
-    index = r_uint(x - lower)
+    index = r_uint(x) - r_uint(lower)
     if index >= r_uint(upper - lower):
         w_res = instantiate(W_IntObject)
     else:
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -371,6 +371,9 @@
     def newmemoryview(self, w_obj):
         return W_MemoryView(w_obj)
 
+    def newmemoryview(self, view):
+        return W_MemoryView(view)
+
     def newbytes(self, s):
         assert isinstance(s, str)
         return W_BytesObject(s)
diff --git a/pypy/tool/release/package.py b/pypy/tool/release/package.py
--- a/pypy/tool/release/package.py
+++ b/pypy/tool/release/package.py
@@ -160,9 +160,9 @@
                 tktcldir = p.dirpath().join('..').join('lib')
                 shutil.copytree(str(tktcldir), str(pypydir.join('tcl')))
             except WindowsError:
-                print >>sys.stderr, """Packaging Tk runtime failed.
-tk85.dll and tcl85.dll found, expecting to find runtime in ..\\lib
-directory next to the dlls, as per build instructions."""
+                print >>sys.stderr, r"""Packaging Tk runtime failed.
+tk85.dll and tcl85.dll found in %s, expecting to find runtime in %s
+directory next to the dlls, as per build instructions.""" %(p, tktcldir)
                 import traceback;traceback.print_exc()
                 raise MissingDependenciesError('Tk runtime')
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -12,7 +12,7 @@
     _immutable_ = True
 
     def getlength(self):
-        """Returns the size in bytes (even if getitemsize() > 1)."""
+        """Return the size in bytes."""
         raise NotImplementedError
 
     def __len__(self):
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -119,6 +119,31 @@
     log.error("Could not find a Microsoft Compiler")
     # Assume that the compiler is already part of the environment
 
+# copied from distutils.spawn
+def _find_executable(executable, path=None):
+    """Tries to find 'executable' in the directories listed in 'path'.
+
+    A string listing directories separated by 'os.pathsep'; defaults to
+    os.environ['PATH'].  Returns the complete filename or None if not found.
+    """
+    if path is None:
+        path = os.environ['PATH']
+    paths = path.split(os.pathsep)
+
+    for ext in '.exe', '':
+        newexe = executable + ext
+
+        if os.path.isfile(newexe):
+            return newexe
+        else:
+            for p in paths:
+                f = os.path.join(p, newexe)
+                if os.path.isfile(f):
+                    # the file exists, we have a shot at spawn working
+                    return f
+    return None
+
+
 class MsvcPlatform(Platform):
     name = "msvc"
     so_ext = 'dll'
@@ -128,6 +153,9 @@
 
     cc = 'cl.exe'
     link = 'link.exe'
+    make = 'nmake'
+    if _find_executable('jom.exe'):
+        make = 'jom.exe'
 
     cflags = ('/MD', '/O2', '/Zi')
     link_flags = ('/debug','/LARGEADDRESSAWARE')
@@ -483,11 +511,11 @@
             path = path_to_makefile.makefile_dir
         else:
             path = path_to_makefile
-        log.execute('make %s in %s' % (" ".join(extra_opts), path))
+        log.execute('%s %s in %s' % (self.make, " ".join(extra_opts), path))
         oldcwd = path.chdir()
         try:
             returncode, stdout, stderr = _run_subprocess(
-                'nmake',
+                self.make,
                 ['/nologo', '/f', str(path.join('Makefile'))] + extra_opts,
                 env = self.c_environ)
         finally:

From pypy.commits at gmail.com  Fri May 12 19:25:43 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 16:25:43 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix bhimpl_gc_store_indexed_i,
 which was not tested because the blackhole didn't see the op :(
Message-ID: <59164477.e587df0a.11692.9eb0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91275:3c31e7d36cc9
Date: 2017-05-12 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/3c31e7d36cc9/

Log:	fix bhimpl_gc_store_indexed_i, which was not tested because the
	blackhole didn't see the op :(

diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1478,12 +1478,17 @@
     def bhimpl_gc_load_indexed_f(cpu, addr, index, scale, base_ofs, bytes):
         return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
 
-    @arguments("cpu", "r", "i", "i", "i", "i", "i")
-    def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs, bytes):
-        return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs, bytes)
-    @arguments("cpu", "r", "i", "f", "i", "i", "i")
-    def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs, bytes):
-        return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs, bytes)
+    @arguments("cpu", "r", "i", "i", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
+
+    @arguments("cpu", "r", "i", "f", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
 
     @arguments("r", "d", "d")
     def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -19,7 +19,8 @@
                   supports_floats=True,
                   supports_longlong=False,
                   supports_singlefloats=False,
-                  translationoptions={}, **kwds):
+                  translationoptions={},
+                  backendopt_inline_threshold=0, **kwds):
     from rpython.jit.codewriter import support
 
     class FakeJitCell(object):
@@ -59,7 +60,7 @@
         FakeWarmRunnerState.enable_opts = {}
 
     func._jit_unroll_safe_ = True
-    rtyper = support.annotate(func, values,
+    rtyper = support.annotate(func, values, inline=backendopt_inline_threshold,
                               translationoptions=translationoptions)
     graphs = rtyper.annotator.translator.graphs
     testself.all_graphs = graphs
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
--- a/rpython/jit/metainterp/test/test_llop.py
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -8,6 +8,7 @@
 from rpython.jit.metainterp.history import getkind
 from rpython.jit.metainterp.test.support import LLJitMixin
 
+
 class TestLLOp(BaseLLOpTest, LLJitMixin):
 
     # for the individual tests see
@@ -37,7 +38,11 @@
                 # I'm not sure why, but if I use an assert, the test doesn't fail
                 raise ValueError('got != expected')
             return len(got)
-        return self.interp_operations(f, [value], supports_singlefloats=True)
+        # we pass a big inline_threshold to ensure that newlist_and_gc_store
+        # is inlined, else the blackhole does not see (and thus we do not
+        # test!) the llop.gc_store_indexed
+        return self.interp_operations(f, [value], supports_singlefloats=True,
+                                      backendopt_inline_threshold=33)
 
 
     def test_force_virtual_str_storage(self):

From pypy.commits at gmail.com  Fri May 12 19:25:45 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 12 May 2017 16:25:45 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: unroll the loop if count is a
 small constant
Message-ID: <59164479.8b121c0a.ccaad.62a7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91276:f99d6f69a91c
Date: 2017-05-13 00:31 +0200
http://bitbucket.org/pypy/pypy/changeset/f99d6f69a91c/

Log:	unroll the loop if count is a small constant

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.annlowlevel import llstr, hlstr
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.buffer import Buffer
+from rpython.rlib import jit
 
 class MutableStringBuffer(Buffer):
     """
@@ -46,6 +47,8 @@
     def setitem(self, index, char):
         self.ll_val.chars[index] = char
 
+    @jit.look_inside_iff(lambda self, index, count:
+                         jit.isconstant(count) and count <= 8)
     def setzeros(self, index, count):
         for i in range(index, index+count):
             self.setitem(i, '\x00')

From pypy.commits at gmail.com  Sat May 13 10:00:19 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 13 May 2017 07:00:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <59171173.5b8bdf0a.31d2d.f8ca@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91277:2157c3d029f1
Date: 2017-05-13 14:59 +0100
http://bitbucket.org/pypy/pypy/changeset/2157c3d029f1/

Log:	fix translation

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -79,7 +79,7 @@
 
     def getname(self, space):
         try:
-            return space.text_w(space.getattr(self, space.newtext('__name__')))
+            return space.unicode_w(space.getattr(self, space.newtext('__name__')))
         except OperationError as e:
             if e.match(space, space.w_TypeError) or e.match(space, space.w_AttributeError):
                 return u'?'

From pypy.commits at gmail.com  Sun May 14 11:25:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 14 May 2017 08:25:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement PyModule_GetState()
Message-ID: <591876ec.cf9adf0a.b7e88.f6e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91278:11e536df3c51
Date: 2017-05-14 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/11e536df3c51/

Log:	Implement PyModule_GetState()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -565,7 +565,7 @@
     'PyUnicode_FromFormat', 'PyUnicode_FromFormatV', 'PyUnicode_AsWideCharString',
     'PyUnicode_GetSize', 'PyUnicode_GetLength',
     'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant',
-    'PyModule_GetDef', 'PyModuleDef_Init',
+    'PyModule_GetDef', 'PyModuleDef_Init', 'PyModule_GetState',
     'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack',
     '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT',
     '_PyArg_ParseTupleAndKeywords_SizeT', '_PyArg_VaParse_SizeT',
diff --git a/pypy/module/cpyext/include/modsupport.h b/pypy/module/cpyext/include/modsupport.h
--- a/pypy/module/cpyext/include/modsupport.h
+++ b/pypy/module/cpyext/include/modsupport.h
@@ -71,6 +71,7 @@
 #define PyModule_AddStringMacro(m, c) PyModule_AddStringConstant(m, #c, c)
 
 PyAPI_FUNC(struct PyModuleDef*) PyModule_GetDef(PyObject*);
+PyAPI_FUNC(void*) PyModule_GetState(PyObject*);
 
 
 PyAPI_FUNC(PyObject *) Py_BuildValue(const char *, ...);
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts,
-    parse_dir, bootstrap_function, generic_cpy_call)
+    parse_dir, bootstrap_function, generic_cpy_call, slot_function)
 from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
@@ -18,7 +18,16 @@
 
 @bootstrap_function
 def init_moduleobject(space):
-    make_typedescr(Module.typedef, basestruct=PyModuleObject.TO)
+    make_typedescr(Module.typedef, basestruct=PyModuleObject.TO,
+                   dealloc=module_dealloc)
+
+ at slot_function([PyObject], lltype.Void)
+def module_dealloc(space, py_obj):
+    py_module = rffi.cast(PyModuleObject, py_obj)
+    if py_module.c_md_state:
+        lltype.free(py_module.c_md_state, flavor='raw')
+    from pypy.module.cpyext.object import _dealloc
+    _dealloc(space, py_obj)
 
 @cpython_api([rffi.CCHARP], PyObject)
 def PyModule_New(space, name):
@@ -49,7 +58,8 @@
     if f_name is not None:
         modname = f_name
     w_mod = Module(space, space.newtext(modname))
-    rffi.cast(PyModuleObject, as_pyobj(space, w_mod)).c_md_def = module
+    py_mod = rffi.cast(PyModuleObject, as_pyobj(space, w_mod))
+    py_mod.c_md_def = module
     state.package_context = None, None
 
     if f_path is not None:
@@ -62,6 +72,10 @@
     if doc:
         space.setattr(w_mod, space.newtext("__doc__"),
                       space.newtext(doc))
+
+    if module.c_m_size > 0:
+        py_mod.c_md_state = lltype.malloc(rffi.VOIDP.TO, module.c_m_size,
+                                          flavor='raw', zero=True)
     return w_mod
 
 
diff --git a/pypy/module/cpyext/parse/cpyext_moduleobject.h b/pypy/module/cpyext/parse/cpyext_moduleobject.h
--- a/pypy/module/cpyext/parse/cpyext_moduleobject.h
+++ b/pypy/module/cpyext/parse/cpyext_moduleobject.h
@@ -40,5 +40,5 @@
 typedef struct {
     PyObject_HEAD
     struct PyModuleDef *md_def;
-    //void *md_state;
+    void *md_state;
 } PyModuleObject;
diff --git a/pypy/module/cpyext/src/modsupport.c b/pypy/module/cpyext/src/modsupport.c
--- a/pypy/module/cpyext/src/modsupport.c
+++ b/pypy/module/cpyext/src/modsupport.c
@@ -603,6 +603,16 @@
     return ((PyModuleObject *)m)->md_def;
 }
 
+void*
+PyModule_GetState(PyObject* m)
+{
+    if (!PyModule_Check(m)) {
+        PyErr_BadArgument();
+        return NULL;
+    }
+    return ((PyModuleObject *)m)->md_state;
+}
+
 PyTypeObject PyModuleDef_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "moduledef",                                /* tp_name */
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -32,6 +32,29 @@
             """)
         assert module.check_getdef_same()
 
+    def test_getstate(self):
+        module = self.import_extension('foo', [
+            ("check_mod_getstate", "METH_NOARGS",
+             """
+                 struct module_state { int foo[51200]; };
+                 static struct PyModuleDef moduledef = {
+                     PyModuleDef_HEAD_INIT,
+                     "module_getstate_myextension",
+                     NULL,
+                     sizeof(struct module_state)
+                 };
+                 PyObject *module = PyModule_Create(&moduledef);
+                 int *p = (int *)PyModule_GetState(module);
+                 int i;
+                 for (i = 0; i < 51200; i++)
+                     if (p[i] != 0)
+                         return PyBool_FromLong(0);
+                 Py_DECREF(module);
+                 return PyBool_FromLong(1);
+             """
+            )])
+        assert module.check_mod_getstate()
+
 
 class AppTestMultiPhase(AppTestCpythonExtensionBase):
     def test_basic(self):

From pypy.commits at gmail.com  Sun May 14 12:04:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 14 May 2017 09:04:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue #2557: file.read(1) could return
 2 bytes on Windows
Message-ID: <59188009.11811c0a.f3d27.cd71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91279:643daedf4ed9
Date: 2017-05-14 18:03 +0200
http://bitbucket.org/pypy/pypy/changeset/643daedf4ed9/

Log:	issue #2557: file.read(1) could return 2 bytes on Windows

diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -902,18 +902,30 @@
         self.do_read = base.read
         self.do_write = base.write
         self.do_flush = base.flush_buffers
-        self.lfbuffer = ""
+        self.readahead_count = 0   # either 0 or 1
 
     def read(self, n=-1):
-        data = self.lfbuffer + self.do_read(n)
-        self.lfbuffer = ""
+        """If n >= 1, this should read between 1 and n bytes."""
+        if n <= 0:
+            if n < 0:
+                return self.readall()
+            else:
+                return ""
+
+        data = self.do_read(n - self.readahead_count)
+        if self.readahead_count > 0:
+            data = self.readahead_char + data
+            self.readahead_count = 0
+
         if data.endswith("\r"):
             c = self.do_read(1)
-            if c and c[0] == '\n':
-                data = data + '\n'
-                self.lfbuffer = c[1:]
-            else:
-                self.lfbuffer = c
+            if len(c) >= 1:
+                assert len(c) == 1
+                if c[0] == '\n':
+                    data = data + '\n'
+                else:
+                    self.readahead_char = c[0]
+                    self.readahead_count = 1
 
         result = []
         offset = 0
@@ -936,21 +948,21 @@
 
     def tell(self):
         pos = self.base.tell()
-        return pos - len(self.lfbuffer)
+        return pos - self.readahead_count
 
     def seek(self, offset, whence):
         if whence == 1:
-            offset -= len(self.lfbuffer)   # correct for already-read-ahead character
+            offset -= self.readahead_count   # correct for already-read-ahead character
         self.base.seek(offset, whence)
-        self.lfbuffer = ""
+        self.readahead_count = 0
 
     def flush_buffers(self):
-        if self.lfbuffer:
+        if self.readahead_count > 0:
             try:
-                self.base.seek(-len(self.lfbuffer), 1)
+                self.base.seek(-self.readahead_count, 1)
             except (MyNotImplementedError, OSError):
                 return
-            self.lfbuffer = ""
+            self.readahead_count = 0
         self.do_flush()
 
     def write(self, data):
diff --git a/rpython/rlib/test/test_streamio.py b/rpython/rlib/test/test_streamio.py
--- a/rpython/rlib/test/test_streamio.py
+++ b/rpython/rlib/test/test_streamio.py
@@ -657,6 +657,23 @@
             assert line == ''
         self.interpret(f, [])
 
+    def test_read1(self):
+        s_input = "abc\r\nabc\nd\r\nef\r\ngha\rbc\rdef\n\r\n\r"
+        s_output = "abc\nabc\nd\nef\ngha\rbc\rdef\n\n\r"
+        assert s_output == s_input.replace('\r\n', '\n')
+        packets = list(s_input)
+        expected = list(s_output)
+        crlf = streamio.TextCRLFFilter(TSource(packets))
+        def f():
+            blocks = []
+            while True:
+                block = crlf.read(1)
+                if not block:
+                    break
+                blocks.append(block)
+            assert blocks == expected
+        self.interpret(f, [])
+
 class TestTextCRLFFilterLLInterp(BaseTestTextCRLFFilter):
     pass
 

From pypy.commits at gmail.com  Sun May 14 19:04:55 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:04:55 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a fastpath for floats and
 doubles
Message-ID: <5918e297.b1b7500a.1fd9c.7ad2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91280:09d1f6469168
Date: 2017-05-13 14:53 +0200
http://bitbucket.org/pypy/pypy/changeset/09d1f6469168/

Log:	add a fastpath for floats and doubles

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -23,6 +23,39 @@
 native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
 
+ at specialize.memo()
+def pack_fastpath(TYPE):
+    """
+    Create a fast path packer for TYPE. The packer returns True is it succeded
+    or False otherwise.
+    """
+    @specialize.argtype(0)
+    def do_pack_fastpath(fmtiter, value):
+        size = rffi.sizeof(TYPE)
+        pos = fmtiter.pos
+        if (not USE_FASTPATH or
+            fmtiter.bigendian != native_is_bigendian or
+            not native_is_ieee754 or
+            pos % size != 0):
+            raise CannotWrite
+        #
+        # typed_write() might raise CannotWrite
+        fmtiter.result.typed_write(TYPE, fmtiter.pos, value)
+        fmtiter.advance(size)
+    #
+    @specialize.argtype(0)
+    def do_pack_fastpath_maybe(fmtiter, value):
+        try:
+            do_pack_fastpath(fmtiter, value)
+        except CannotWrite:
+            if not ALLOW_SLOWPATH:
+                raise ValueErro("fastpath not taken :(")
+            return False
+        else:
+            return True
+    #
+    return do_pack_fastpath_maybe
+
 def pack_pad(fmtiter, count):
     fmtiter.result.setzeros(fmtiter.pos, count)
     fmtiter.advance(count)
@@ -68,24 +101,14 @@
     fmtiter.advance(1)
     _pack_string(fmtiter, string, count-1)
 
- at specialize.memo()
-def pack_fastpath(TYPE):
-    @specialize.argtype(0)
-    def do_pack_fastpath(fmtiter, value):
-        size = rffi.sizeof(TYPE)
-        pos = fmtiter.pos
-        if (not USE_FASTPATH or
-            fmtiter.bigendian != native_is_bigendian or
-            pos % size != 0):
-            raise CannotWrite
-        # the following might raise CannotWrite and abort the fastpath
-        fmtiter.result.typed_write(TYPE, fmtiter.pos, value)
-        fmtiter.advance(size)
-    return do_pack_fastpath
 
-def make_float_packer(size):
+def make_float_packer(TYPE):
+    size = rffi.sizeof(TYPE)
     def packer(fmtiter):
         fl = fmtiter.accept_float_arg()
+        if pack_fastpath(TYPE)(fmtiter, fl):
+            return
+        # slow path
         try:
             result = ieee.pack_float(fmtiter.result, fmtiter.pos,
                                      fl, size, fmtiter.bigendian)
@@ -150,13 +173,8 @@
         if not min <= value <= max:
             raise StructError(errormsg)
         #
-        try:
-            pack_fastpath(TYPE)(fmtiter, value)
+        if pack_fastpath(TYPE)(fmtiter, value):
             return
-        except CannotWrite:
-            if not ALLOW_SLOWPATH:
-                # we enter here only in some tests
-                raise ValueError("fastpath not taken :(")
         #
         pos = fmtiter.pos + size - 1        
         if fmtiter.bigendian:
@@ -303,7 +321,7 @@
 
     @specialize.argtype(0)
     def unpack_int_fastpath_maybe(fmtiter):
-        if fmtiter.bigendian != native_is_bigendian or not native_is_ieee754: ## or not str_storage_supported(TYPE):
+        if fmtiter.bigendian != native_is_bigendian or not native_is_ieee754:
             return False
         try:
             intvalue = unpack_fastpath(TYPE)(fmtiter)
@@ -356,9 +374,9 @@
           'needcount' : True },
     'p':{ 'size' : 1, 'pack' : pack_pascal, 'unpack' : unpack_pascal,
           'needcount' : True },
-    'f':{ 'size' : 4, 'pack' : make_float_packer(4),
+    'f':{ 'size' : 4, 'pack' : make_float_packer(rffi.FLOAT),
                     'unpack' : unpack_float},
-    'd':{ 'size' : 8, 'pack' : make_float_packer(8),
+    'd':{ 'size' : 8, 'pack' : make_float_packer(rffi.DOUBLE),
                     'unpack' : unpack_double},
     '?':{ 'size' : 1, 'pack' : pack_bool, 'unpack' : unpack_bool},
     }

From pypy.commits at gmail.com  Sun May 14 19:05:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: pass an external wbuf to
 PackFormatIterator
Message-ID: <5918e29d.a690500a.791dc.4569@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91283:00ab2f34b4af
Date: 2017-05-13 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/00ab2f34b4af/

Log:	pass an external wbuf to PackFormatIterator

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -2,7 +2,6 @@
                                       maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 
@@ -10,12 +9,12 @@
 
 
 class PackFormatIterator(FormatIterator):
-    def __init__(self, space, args_w, size):
+    def __init__(self, space, wbuf, args_w):
         self.space = space
         self.args_w = args_w
         self.args_index = 0
         self.pos = 0
-        self.wbuf = MutableStringBuffer(size)
+        self.wbuf = wbuf
 
     def advance(self, count):
         self.pos += count
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,5 +1,6 @@
 from rpython.rlib import jit
 from rpython.rlib.buffer import SubBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
@@ -41,15 +42,16 @@
 def _pack(space, format, args_w):
     """Return string containing values v1, v2, ... packed according to fmt."""
     size = _calcsize(space, format)
-    fmtiter = PackFormatIterator(space, args_w, size)
+    wbuf = MutableStringBuffer(size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
         raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
         raise OperationError(get_error(space), space.newtext(e.msg))
-    assert fmtiter.pos == fmtiter.wbuf.getlength(), 'missing .advance() or wrong calcsize()'
-    return fmtiter.wbuf.finish()
+    assert fmtiter.pos == wbuf.getlength(), 'missing .advance() or wrong calcsize()'
+    return wbuf.finish()
 
 
 @unwrap_spec(format='text')

From pypy.commits at gmail.com  Sun May 14 19:04:57 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:04:57 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: use the fast path also for the
 native float/double packing
Message-ID: <5918e299.0990500a.5eb9f.8abe@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91281:38cd23755b61
Date: 2017-05-13 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/38cd23755b61/

Log:	use the fast path also for the native float/double packing

diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -29,6 +29,9 @@
 
 def pack_double(fmtiter):
     doubleval = fmtiter.accept_float_arg()
+    if std.pack_fastpath(rffi.DOUBLE)(fmtiter, doubleval):
+        return
+    # slow path
     value = longlong2float.float2longlong(doubleval)
     pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 8, fmtiter.bigendian)
     fmtiter.advance(8)
@@ -36,6 +39,9 @@
 def pack_float(fmtiter):
     doubleval = fmtiter.accept_float_arg()
     floatval = r_singlefloat(doubleval)
+    if std.pack_fastpath(rffi.FLOAT)(fmtiter, floatval):
+        return
+    # slow path
     value = longlong2float.singlefloat2uint(floatval)
     value = widen(value)
     pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 4, fmtiter.bigendian)
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -144,3 +144,9 @@
     bigendian = nativefmttable.native_is_bigendian
     fmt_prefix = '@'
     fmttable = nativefmttable.native_fmttable
+
+class TestNativeSlowPath(BaseTestPack):
+    USE_FASTPATH = False
+    bigendian = nativefmttable.native_is_bigendian
+    fmt_prefix = '@'
+    fmttable = nativefmttable.native_fmttable

From pypy.commits at gmail.com  Sun May 14 19:05:05 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:05 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: whoo... finally reach the
 whole point of the branch: struct.pack_into(bytearray...) takes the fast
 path :)
Message-ID: <5918e2a1.6d91500a.4b115.4f4f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91285:973fa84efaef
Date: 2017-05-13 15:39 +0200
http://bitbucket.org/pypy/pypy/changeset/973fa84efaef/

Log:	whoo... finally reach the whole point of the branch:
	struct.pack_into(bytearray...) takes the fast path :)

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -518,3 +518,10 @@
         data = self.struct.pack("iii", 0, 42, 43)
         buf = array.array('c', data)
         assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_pack_into_bytearray(self):
+        expected = self.struct.pack("ii", 42, 43)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("ii", buf, 0, 42, 43)
+        assert buf == expected
+        
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1252,18 +1252,31 @@
     def get_raw_address(self):
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
-    @specialize.ll_and_arg(1)
-    def typed_read(self, TP, byte_offset):
+    def _get_gc_data(self):
         from rpython.rtyper.lltypesystem import lltype, llmemory
-        from rpython.rtyper.lltypesystem.lloperation import llop
         ll_data = ll_for_resizable_list(self.data)
         ll_items = ll_data.items
         LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
         base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
         scale_factor = llmemory.sizeof(lltype.Char)
+        return ll_items, scale_factor, base_ofs
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        ll_items, scale_factor, base_ofs = self._get_gc_data()
         return llop.gc_load_indexed(TP, ll_items, byte_offset,
                                     scale_factor, base_ofs)
 
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        from rpython.rtyper.lltypesystem import lltype
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        ll_items, scale_factor, base_ofs = self._get_gc_data()
+        value = lltype.cast_primitive(TP, value)
+        return llop.gc_store_indexed(lltype.Void, ll_items, byte_offset, value,
+                                     scale_factor, base_ofs)
+
 
 
 @specialize.argtype(1)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -240,3 +240,7 @@
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
         return self.buffer.typed_read(TP, byte_offset + self.offset)
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        return self.buffer.typed_write(TP, byte_offset + self.offset, value)
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -49,7 +49,7 @@
             do_pack_fastpath(fmtiter, value)
         except CannotWrite:
             if not ALLOW_SLOWPATH:
-                raise ValueErro("fastpath not taken :(")
+                raise ValueError("fastpath not taken :(")
             return False
         else:
             return True

From pypy.commits at gmail.com  Sun May 14 19:04:59 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:04:59 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rename fmtiter.result into
 wbuf, which makes more sense
Message-ID: <5918e29b.6997500a.fe174.5ba0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91282:91e0aa0b6c31
Date: 2017-05-13 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/91e0aa0b6c31/

Log:	rename fmtiter.result into wbuf, which makes more sense

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -15,7 +15,7 @@
         self.args_w = args_w
         self.args_index = 0
         self.pos = 0
-        self.result = MutableStringBuffer(size)
+        self.wbuf = MutableStringBuffer(size)
 
     def advance(self, count):
         self.pos += count
@@ -36,7 +36,7 @@
     @jit.unroll_safe
     def align(self, mask):
         pad = (-self.pos) & mask
-        self.result.setzeros(self.pos, pad)
+        self.wbuf.setzeros(self.pos, pad)
         self.advance(pad)
 
     def finished(self):
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -48,8 +48,8 @@
         raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
         raise OperationError(get_error(space), space.newtext(e.msg))
-    assert fmtiter.pos == fmtiter.result.getlength(), 'missing .advance() or wrong calcsize()'
-    return fmtiter.result.finish()
+    assert fmtiter.pos == fmtiter.wbuf.getlength(), 'missing .advance() or wrong calcsize()'
+    return fmtiter.wbuf.finish()
 
 
 @unwrap_spec(format='text')
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -265,22 +265,22 @@
     return (mant, (sign << BITS - MANT_DIG - 1) | exp)
 
 
-def pack_float(result, pos, x, size, be):
+def pack_float(wbuf, pos, x, size, be):
     unsigned = float_pack(x, size)
     value = rarithmetic.longlongmask(unsigned)
-    pack_float_to_buffer(result, pos, value, size, be)
+    pack_float_to_buffer(wbuf, pos, value, size, be)
 
 @jit.unroll_safe
-def pack_float_to_buffer(result, pos, value, size, be):
+def pack_float_to_buffer(wbuf, pos, value, size, be):
     if be:
         # write in reversed order
         for i in range(size):
             c = chr((value >> (i * 8)) & 0xFF)
-            result.setitem(pos + size - i - 1, c)
+            wbuf.setitem(pos + size - i - 1, c)
     else:
         for i in range(size):
             c = chr((value >> (i * 8)) & 0xFF)
-            result.setitem(pos+i, c)
+            wbuf.setitem(pos+i, c)
 
 @jit.unroll_safe
 def pack_float80(result, x, size, be):
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -33,7 +33,7 @@
         return
     # slow path
     value = longlong2float.float2longlong(doubleval)
-    pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 8, fmtiter.bigendian)
+    pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 8, fmtiter.bigendian)
     fmtiter.advance(8)
 
 def pack_float(fmtiter):
@@ -44,7 +44,7 @@
     # slow path
     value = longlong2float.singlefloat2uint(floatval)
     value = widen(value)
-    pack_float_to_buffer(fmtiter.result, fmtiter.pos, value, 4, fmtiter.bigendian)
+    pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 4, fmtiter.bigendian)
     fmtiter.advance(4)
 
 # ____________________________________________________________
@@ -142,7 +142,7 @@
     if len(unistr) != 1:
         raise StructError("expected a unicode string of length 1")
     c = unistr[0]   # string->char conversion for the annotator
-    unichar.pack_unichar(c, fmtiter.result, fmtiter.pos)
+    unichar.pack_unichar(c, fmtiter.wbuf, fmtiter.pos)
     fmtiter.advance(unichar.UNICODE_SIZE)
 
 @specialize.argtype(0)
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -40,7 +40,7 @@
             raise CannotWrite
         #
         # typed_write() might raise CannotWrite
-        fmtiter.result.typed_write(TYPE, fmtiter.pos, value)
+        fmtiter.wbuf.typed_write(TYPE, fmtiter.pos, value)
         fmtiter.advance(size)
     #
     @specialize.argtype(0)
@@ -57,7 +57,7 @@
     return do_pack_fastpath_maybe
 
 def pack_pad(fmtiter, count):
-    fmtiter.result.setzeros(fmtiter.pos, count)
+    fmtiter.wbuf.setzeros(fmtiter.pos, count)
     fmtiter.advance(count)
 
 def pack_char(fmtiter):
@@ -65,23 +65,23 @@
     if len(string) != 1:
         raise StructError("expected a string of length 1")
     c = string[0]   # string->char conversion for the annotator
-    fmtiter.result.setitem(fmtiter.pos, c)
+    fmtiter.wbuf.setitem(fmtiter.pos, c)
     fmtiter.advance(1)
 
 def pack_bool(fmtiter):
     c = '\x01' if fmtiter.accept_bool_arg() else '\x00'
-    fmtiter.result.setitem(fmtiter.pos, c)
+    fmtiter.wbuf.setitem(fmtiter.pos, c)
     fmtiter.advance(1)
 
 def _pack_string(fmtiter, string, count):
     pos = fmtiter.pos
     if len(string) < count:
         n = len(string)
-        fmtiter.result.setslice(pos, string)
-        fmtiter.result.setzeros(pos+n, count-n)
+        fmtiter.wbuf.setslice(pos, string)
+        fmtiter.wbuf.setzeros(pos+n, count-n)
     else:
         assert count >= 0
-        fmtiter.result.setslice(pos, string[:count])
+        fmtiter.wbuf.setslice(pos, string[:count])
     fmtiter.advance(count)
 
 def pack_string(fmtiter, count):
@@ -97,7 +97,7 @@
             raise StructError("bad '0p' in struct format")
     if prefix > 255:
         prefix = 255
-    fmtiter.result.setitem(fmtiter.pos, chr(prefix))
+    fmtiter.wbuf.setitem(fmtiter.pos, chr(prefix))
     fmtiter.advance(1)
     _pack_string(fmtiter, string, count-1)
 
@@ -110,7 +110,7 @@
             return
         # slow path
         try:
-            result = ieee.pack_float(fmtiter.result, fmtiter.pos,
+            result = ieee.pack_float(fmtiter.wbuf, fmtiter.pos,
                                      fl, size, fmtiter.bigendian)
         except OverflowError:
             assert size == 4
@@ -180,11 +180,11 @@
         if fmtiter.bigendian:
             for i in unroll_revrange_size:
                 x = (value >> (8*i)) & 0xff
-                fmtiter.result.setitem(pos-i, chr(x))
+                fmtiter.wbuf.setitem(pos-i, chr(x))
         else:
 
             for i in unroll_revrange_size:
-                fmtiter.result.setitem(pos-i, chr(value & 0xff))
+                fmtiter.wbuf.setitem(pos-i, chr(value & 0xff))
                 value >>= 8
         fmtiter.advance(size)
 
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -10,7 +10,7 @@
         from rpython.rlib.rstring import StringBuilder
         self.value = value
         self.bigendian = bigendian
-        self.result = MutableStringBuffer(size)
+        self.wbuf = MutableStringBuffer(size)
         self.pos = 0
 
     def advance(self, count):
@@ -18,8 +18,8 @@
 
     def finish(self):
         # check that we called advance() the right number of times
-        assert self.pos == self.result.getlength()
-        return self.result.finish()
+        assert self.pos == self.wbuf.getlength()
+        return self.wbuf.finish()
 
     def _accept_arg(self):
         return self.value

From pypy.commits at gmail.com  Sun May 14 19:05:07 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:07 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: we cannot use the fastpath for
 '<f' or '>f',
 because we need to check for overflow in that case. It is fine to use the
 fastpath for doubles though, because there is no risk of overflow
Message-ID: <5918e2a3.33ba500a.2b1ce.a489@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91286:6c228d591ce8
Date: 2017-05-15 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/6c228d591ce8/

Log:	we cannot use the fastpath for '<f' or '>f', because we need to
	check for overflow in that case. It is fine to use the fastpath for
	doubles though, because there is no risk of overflow

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -106,7 +106,7 @@
     size = rffi.sizeof(TYPE)
     def packer(fmtiter):
         fl = fmtiter.accept_float_arg()
-        if pack_fastpath(TYPE)(fmtiter, fl):
+        if TYPE is not rffi.FLOAT and pack_fastpath(TYPE)(fmtiter, fl):
             return
         # slow path
         try:
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -1,6 +1,7 @@
 import pytest
 from rpython.rlib.rarithmetic import r_ulonglong
 from rpython.rlib.rstruct import standardfmttable, nativefmttable
+from rpython.rlib.rstruct.error import StructOverflowError
 from rpython.rlib.mutbuffer import MutableStringBuffer
 import struct
 
@@ -93,6 +94,14 @@
         self.check('f', 123.456)
         self.check('d', 123.456789)
 
+    def test_float_overflow(self):
+        if self.fmt_prefix == '@':
+            # native packing, no overflow
+            self.check('f', 10e100)
+        else:
+            # non-native packing, should raise
+            pytest.raises(StructOverflowError, "self.mypack('f', 10e100)")
+
     def test_pack_char(self):
         self.check('c', 'a')
 

From pypy.commits at gmail.com  Sun May 14 19:05:09 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:09 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix the tests in
 jit/backend/x86/test/test_llop.py:test_gc_store*
Message-ID: <5918e2a5.d295500a.9ce57.536d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91287:e18391926354
Date: 2017-05-15 00:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e18391926354/

Log:	fix the tests in jit/backend/x86/test/test_llop.py:test_gc_store*

diff --git a/rpython/jit/backend/x86/test/test_llop.py b/rpython/jit/backend/x86/test/test_llop.py
--- a/rpython/jit/backend/x86/test/test_llop.py
+++ b/rpython/jit/backend/x86/test/test_llop.py
@@ -5,5 +5,10 @@
 class TestLLOp(Jit386Mixin, _TestLLOp):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_llop.py
-    pass
 
+    # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+    # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+    # symbolic as the offset. It is not a problem because it is tested anyway
+    # by the same test in test_metainterp.py
+    TEST_BLACKHOLE = False
+
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
--- a/rpython/jit/metainterp/test/test_llop.py
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -13,6 +13,7 @@
 
     # for the individual tests see
     # ====> ../../../rtyper/test/test_llop.py
+    TEST_BLACKHOLE = True
 
     def gc_load_from_string(self, TYPE, buf, offset):
         def f(offset):
@@ -38,11 +39,16 @@
                 # I'm not sure why, but if I use an assert, the test doesn't fail
                 raise ValueError('got != expected')
             return len(got)
-        # we pass a big inline_threshold to ensure that newlist_and_gc_store
-        # is inlined, else the blackhole does not see (and thus we do not
-        # test!) the llop.gc_store_indexed
+        #
+        if self.TEST_BLACKHOLE:
+            # we pass a big inline_threshold to ensure that
+            # newlist_and_gc_store is inlined, else the blackhole does not see
+            # (and thus we do not test!) the llop.gc_store_indexed
+            threshold = 33
+        else:
+            threshold = 0
         return self.interp_operations(f, [value], supports_singlefloats=True,
-                                      backendopt_inline_threshold=33)
+                                      backendopt_inline_threshold=threshold)
 
 
     def test_force_virtual_str_storage(self):

From pypy.commits at gmail.com  Sun May 14 19:05:03 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:03 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: now that we have all the
 necessary infrastructure, implement pack_into in a more efficient way,
 so that PackFormatIterator writes directly inside the destination buffer
 (using the fast paths if possible)
Message-ID: <5918e29f.bb87500a.1f2df.38b7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91284:a8366709a997
Date: 2017-05-13 15:27 +0200
http://bitbucket.org/pypy/pypy/changeset/a8366709a997/

Log:	now that we have all the necessary infrastructure, implement
	pack_into in a more efficient way, so that PackFormatIterator writes
	directly inside the destination buffer (using the fast paths if
	possible)

diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -35,7 +35,8 @@
     @jit.unroll_safe
     def align(self, mask):
         pad = (-self.pos) & mask
-        self.wbuf.setzeros(self.pos, pad)
+        for i in range(self.pos, self.pos+pad):
+            self.wbuf.setitem(i, '\x00')
         self.advance(pad)
 
     def finished(self):
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -59,22 +59,28 @@
     return space.newbytes(_pack(space, format, args_w))
 
 
-# XXX inefficient
 @unwrap_spec(format='text', offset=int)
 def pack_into(space, format, w_buffer, offset, args_w):
     """ Pack the values v1, v2, ... according to fmt.
 Write the packed bytes into the writable buffer buf starting at offset
     """
-    res = _pack(space, format, args_w)
+    size = _calcsize(space, format)
     buf = space.getarg_w('w*', w_buffer)
     if offset < 0:
         offset += buf.getlength()
-    size = len(res)
     if offset < 0 or (buf.getlength() - offset) < size:
         raise oefmt(get_error(space),
                     "pack_into requires a buffer of at least %d bytes",
                     size)
-    buf.setslice(offset, res)
+    #
+    wbuf = SubBuffer(buf, offset, size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
+    try:
+        fmtiter.interpret(format)
+    except StructOverflowError as e:
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
+    except StructError as e:
+        raise OperationError(get_error(space), space.newtext(e.msg))
 
 
 def _unpack(space, format, buf):

From pypy.commits at gmail.com  Sun May 14 19:05:14 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:14 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add tests for the fast path of
 pack_into
Message-ID: <5918e2aa.0990500a.5eb9f.8ac5@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91290:b3c274e5c60c
Date: 2017-05-15 01:03 +0200
http://bitbucket.org/pypy/pypy/changeset/b3c274e5c60c/

Log:	add tests for the fast path of pack_into

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -146,3 +146,48 @@
             guard_not_invalidated(descr=...)
             i70 = gc_load_indexed_i(p48, 0, 1, _, -2)
         """)
+
+    def test_pack_into_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = array.array('b', '\x00'*8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            i65 = int_le(i58, 32767)
+            guard_true(i65, descr=...)
+            raw_store(i55, 4, i58, descr=<ArrayS 2>)
+        """)
+
+    def test_pack_into_bytearray(self):
+        def main(n):
+            import struct
+            buf = bytearray(8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            p66 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst_data \d+>)
+            i67 = getfield_gc_i(p66, descr=<FieldS list.length \d+>)
+            i69 = int_sub(i67, 4)
+            i71 = int_lt(i69, 2)
+            guard_false(i71, descr=...)
+            i73 = int_le(i60, 32767)
+            guard_true(i73, descr=...)
+            p74 = getfield_gc_r(p66, descr=<FieldP list.items \d+>)
+            gc_store_indexed(p74, 4, i60, 1, _, 2, descr=<ArrayS 2>)
+        """)

From pypy.commits at gmail.com  Sun May 14 19:05:10 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:10 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix the test_pypy_c tests
 about the struct module, now that we have the fast path also for packing
Message-ID: <5918e2a6.b1b7500a.1fd9c.7ad5@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91288:1bffd7929b3f
Date: 2017-05-15 00:43 +0200
http://bitbucket.org/pypy/pypy/changeset/1bffd7929b3f/

Log:	fix the test_pypy_c tests about the struct module, now that we have
	the fast path also for packing

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -30,32 +30,33 @@
         loop, = log.loops_by_filename(self.filepath)
         # This could, of course stand some improvement, to remove all these
         # arithmatic ops, but we've removed all the core overhead.
-        assert loop.match_by_id("pack", """
-            guard_not_invalidated(descr=...)
-            # struct.pack
-            %s
-            i11 = int_and(i4, 255)
-            i13 = int_rshift(i4, 8)
-            i14 = int_and(i13, 255)
-            i16 = int_rshift(i13, 8)
-            i17 = int_and(i16, 255)
-            i19 = int_rshift(i16, 8)
-            i20 = int_and(i19, 255)
-        """ % extra)
+        if sys.byteorder == 'little':
+            # on little endian machines, we take the fast path and store the
+            # value using gc_store_indexed
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                p75 = newstr(4)
+                gc_store_indexed(p75, 0, _, 1, _, 4, descr=...)
+            """ % extra)
+        else:
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                i11 = int_and(i4, 255)
+                i13 = int_rshift(i4, 8)
+                i14 = int_and(i13, 255)
+                i16 = int_rshift(i13, 8)
+                i17 = int_and(i16, 255)
+                i19 = int_rshift(i16, 8)
+                i20 = int_and(i19, 255)
+            """ % extra)
 
         if sys.byteorder == 'little':
-            # the newstr and the strsetitems are because the string is forced,
-            # which is in turn because the optimizer doesn't know how to handle a
-            # gc_load_indexed_i on a virtual string. It could be improved, but it
-            # is also true that in real life cases struct.unpack is called on
-            # strings which come from the outside, so it's a minor issue.
             assert loop.match_by_id("unpack", """
                 # struct.unpack
-                p88 = newstr(4)
-                strsetitem(p88, 0, i11)
-                strsetitem(p88, 1, i14)
-                strsetitem(p88, 2, i17)
-                strsetitem(p88, 3, i20)
                 i91 = gc_load_indexed_i(p88, 0, 1, _, -4)
             """)
         else:
@@ -93,27 +94,14 @@
             assert loop.match_by_id('pack', """
                 guard_not_invalidated(descr=...)
                 # struct.pack
+                p85 = newstr(8)
+                gc_store_indexed(p85, 0, -1, 1, _, 4, descr=...)
                 %s
-                i11 = int_and(i4, 255)
-                i13 = int_rshift(i4, 8)
-                i14 = int_and(i13, 255)
-                i16 = int_rshift(i13, 8)
-                i17 = int_and(i16, 255)
-                i19 = int_rshift(i16, 8)
-                i20 = int_and(i19, 255)
+                gc_store_indexed(p85, 4, _, 1, _, 4, descr=...)
             """ % extra)
 
             assert loop.match_by_id('unpack', """
                 # struct.unpack
-                p88 = newstr(8)
-                strsetitem(p88, 0, 255)
-                strsetitem(p88, 1, 255)
-                strsetitem(p88, 2, 255)
-                strsetitem(p88, 3, 255)
-                strsetitem(p88, 4, i11)
-                strsetitem(p88, 5, i14)
-                strsetitem(p88, 6, i17)
-                strsetitem(p88, 7, i20)
                 i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
                 i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
             """)

From pypy.commits at gmail.com  Sun May 14 19:05:12 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 14 May 2017 16:05:12 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add tests for the fast paths
 of struct.unpack on raw buffers and bytearrays
Message-ID: <5918e2a8.18d3500a.caa88.225f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91289:4c03a04236a1
Date: 2017-05-15 00:55 +0200
http://bitbucket.org/pypy/pypy/changeset/4c03a04236a1/

Log:	add tests for the fast paths of struct.unpack on raw buffers and
	bytearrays

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -105,3 +105,44 @@
                 i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
                 i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
             """)
+
+    def test_unpack_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = array.array('b', buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i65 = raw_load_i(i49, 0, descr=<ArrayS 2>)
+        """)
+
+    def test_unpack_bytearray(self):
+        def main(n):
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = bytearray(buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i70 = gc_load_indexed_i(p48, 0, 1, _, -2)
+        """)

From pypy.commits at gmail.com  Sun May 14 21:21:14 2017
From: pypy.commits at gmail.com (ltratt)
Date: Sun, 14 May 2017 18:21:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: OpenBSD also needs sys/ttycom.h
 included.
Message-ID: <5919028a.0d91500a.763fb.238b@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91291:51b52e05a32a
Date: 2017-05-14 19:22 +0800
http://bitbucket.org/pypy/pypy/changeset/51b52e05a32a/

Log:	OpenBSD also needs sys/ttycom.h included.

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -239,7 +239,7 @@
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
         includes.append('sys/sysmacros.h')
-    if sys.platform.startswith('freebsd'):
+    if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'):
         includes.append('sys/ttycom.h')
     libraries = ['util']
 eci = ExternalCompilationInfo(

From pypy.commits at gmail.com  Sun May 14 21:21:16 2017
From: pypy.commits at gmail.com (ltratt)
Date: Sun, 14 May 2017 18:21:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: string.h needs to be included for
 strlen to be found.
Message-ID: <5919028c.abb3500a.2bad0.a55e@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91292:42c6ee223963
Date: 2017-05-14 19:25 +0800
http://bitbucket.org/pypy/pypy/changeset/42c6ee223963/

Log:	string.h needs to be included for strlen to be found.

diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c
--- a/rpython/rlib/rvmprof/src/shared/machine.c
+++ b/rpython/rlib/rvmprof/src/shared/machine.c
@@ -4,6 +4,7 @@
 #include <stdio.h>
 
 #ifdef VMPROF_UNIX
+#include <string.h>
 #include <unistd.h>
 #include <fcntl.h>
 #endif

From pypy.commits at gmail.com  Sun May 14 21:21:20 2017
From: pypy.commits at gmail.com (ltratt)
Date: Sun, 14 May 2017 18:21:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge default
Message-ID: <59190290.0d91500a.763fb.238c@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91294:92e51c0101b5
Date: 2017-05-15 09:18 +0800
http://bitbucket.org/pypy/pypy/changeset/92e51c0101b5/

Log:	hg merge default

diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -902,18 +902,30 @@
         self.do_read = base.read
         self.do_write = base.write
         self.do_flush = base.flush_buffers
-        self.lfbuffer = ""
+        self.readahead_count = 0   # either 0 or 1
 
     def read(self, n=-1):
-        data = self.lfbuffer + self.do_read(n)
-        self.lfbuffer = ""
+        """If n >= 1, this should read between 1 and n bytes."""
+        if n <= 0:
+            if n < 0:
+                return self.readall()
+            else:
+                return ""
+
+        data = self.do_read(n - self.readahead_count)
+        if self.readahead_count > 0:
+            data = self.readahead_char + data
+            self.readahead_count = 0
+
         if data.endswith("\r"):
             c = self.do_read(1)
-            if c and c[0] == '\n':
-                data = data + '\n'
-                self.lfbuffer = c[1:]
-            else:
-                self.lfbuffer = c
+            if len(c) >= 1:
+                assert len(c) == 1
+                if c[0] == '\n':
+                    data = data + '\n'
+                else:
+                    self.readahead_char = c[0]
+                    self.readahead_count = 1
 
         result = []
         offset = 0
@@ -936,21 +948,21 @@
 
     def tell(self):
         pos = self.base.tell()
-        return pos - len(self.lfbuffer)
+        return pos - self.readahead_count
 
     def seek(self, offset, whence):
         if whence == 1:
-            offset -= len(self.lfbuffer)   # correct for already-read-ahead character
+            offset -= self.readahead_count   # correct for already-read-ahead character
         self.base.seek(offset, whence)
-        self.lfbuffer = ""
+        self.readahead_count = 0
 
     def flush_buffers(self):
-        if self.lfbuffer:
+        if self.readahead_count > 0:
             try:
-                self.base.seek(-len(self.lfbuffer), 1)
+                self.base.seek(-self.readahead_count, 1)
             except (MyNotImplementedError, OSError):
                 return
-            self.lfbuffer = ""
+            self.readahead_count = 0
         self.do_flush()
 
     def write(self, data):
diff --git a/rpython/rlib/test/test_streamio.py b/rpython/rlib/test/test_streamio.py
--- a/rpython/rlib/test/test_streamio.py
+++ b/rpython/rlib/test/test_streamio.py
@@ -657,6 +657,23 @@
             assert line == ''
         self.interpret(f, [])
 
+    def test_read1(self):
+        s_input = "abc\r\nabc\nd\r\nef\r\ngha\rbc\rdef\n\r\n\r"
+        s_output = "abc\nabc\nd\nef\ngha\rbc\rdef\n\n\r"
+        assert s_output == s_input.replace('\r\n', '\n')
+        packets = list(s_input)
+        expected = list(s_output)
+        crlf = streamio.TextCRLFFilter(TSource(packets))
+        def f():
+            blocks = []
+            while True:
+                block = crlf.read(1)
+                if not block:
+                    break
+                blocks.append(block)
+            assert blocks == expected
+        self.interpret(f, [])
+
 class TestTextCRLFFilterLLInterp(BaseTestTextCRLFFilter):
     pass
 

From pypy.commits at gmail.com  Sun May 14 21:21:18 2017
From: pypy.commits at gmail.com (ltratt)
Date: Sun, 14 May 2017 18:21:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: Disable vmprof on OpenBSD as it doesn't
 build.
Message-ID: <5919028e.3c86500a.ab59b.5566@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91293:00193a29fff8
Date: 2017-05-15 09:10 +0800
http://bitbucket.org/pypy/pypy/changeset/00193a29fff8/

Log:	Disable vmprof on OpenBSD as it doesn't build.

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -42,8 +42,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 

From pypy.commits at gmail.com  Mon May 15 08:27:09 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 05:27:09 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: hg merge default: lots of
 conflicts due to the merge of Pybuffer-backport: textually fixed the
 conflicts but did not run the tests yet
Message-ID: <59199e9d.72b6500a.ad5eb.ce94@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91295:35bd6d4780d4
Date: 2017-05-15 11:30 +0200
http://bitbucket.org/pypy/pypy/changeset/35bd6d4780d4/

Log:	WIP: hg merge default: lots of conflicts due to the merge of
	Pybuffer-backport: textually fixed the conflicts but did not run the
	tests yet

diff too long, truncating to 2000 out of 3662 lines

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,14 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,8 @@
 .. branch: controller-refactor
 
 Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (for post-5.7.1 builds) with sha256 checksum 
-``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -128,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -236,6 +236,9 @@
     copy out32\*.lib <somewhere in LIB>
     xcopy /S include\openssl <somewhere in INCLUDE>
 
+For tests you will also need the dlls::
+    nmake -f ms\ntdll.mak install
+    copy out32dll\*.dll <somewhere in PATH>
 
 TkInter module support
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -245,18 +248,17 @@
 directory found for the release script, create the dlls, libs, headers and
 runtime by running::
 
-	svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
-	svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
-	cd tcl85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
-	nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
-	cd ..\..\tk85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+    svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+    svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+    cd tcl85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
+    nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+    cd ..\..\tk85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+    copy ..\..\tcltk\bin\* <somewhere in PATH>
+    copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+    xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
 
 The lzma compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -341,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,9 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.buffer import StringBuffer
 
+from pypy.interpreter.buffer import BufferInterfaceNotFound
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -221,7 +223,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
 
@@ -233,7 +236,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -245,7 +249,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -254,7 +259,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -392,9 +398,6 @@
 class DescrMismatch(Exception):
     pass
 
-class BufferInterfaceNotFound(Exception):
-    pass
-
 @specialize.memo()
 def wrappable_class_name(Class):
     try:
@@ -1501,18 +1504,28 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+        except OperationError:
+            self._getarg_error("convertible to a buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.readbuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a readable buffer object")
+            self._getarg_error("convertible to a buffer", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+        except OperationError:
+            self._getarg_error("read-write buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.writebuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a writeable buffer object")
+            self._getarg_error("read-write buffer", w_obj)
 
     def charbuf_w(self, w_obj):
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
@@ -1541,12 +1554,10 @@
             if self.isinstance_w(w_obj, self.w_unicode):
                 return self.str(w_obj).readbuf_w(self)
             try:
-                return w_obj.buffer_w(self, 0)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.readbuf_w(self)
-            except BufferInterfaceNotFound:
+                return self.readbuf_w(w_obj)
+            except OperationError as e:
+                if not e.match(self, self.w_TypeError):
+                    raise
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_bytes):
@@ -1558,16 +1569,7 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
-            try:
-                return w_obj.buffer_w(self, self.BUF_WRITABLE)
-            except OperationError:
-                self._getarg_error("read-write buffer", w_obj)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.writebuf_w(self)
-            except BufferInterfaceNotFound:
-                self._getarg_error("read-write buffer", w_obj)
+            return self.writebuf_w(w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
@@ -1654,6 +1656,23 @@
     def fsencode_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
 
+    def byte_w(self, w_obj):
+        """
+        Convert an index-like object to an interp-level char
+
+        Used for app-level code like "bytearray(b'abc')[0] = 42".
+        """
+        if self.isinstance_w(w_obj, self.w_bytes):
+            string = self.bytes_w(w_obj)
+            if len(string) != 1:
+                raise oefmt(self.w_ValueError, "string must be of size 1")
+            return string[0]
+        value = self.getindex_w(w_obj, None)
+        if not 0 <= value < 256:
+            # this includes the OverflowError in case the long is too large
+            raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
+        return chr(value)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,295 @@
+from rpython.rlib.buffer import StringBuffer, SubBuffer
+
+from pypy.interpreter.error import oefmt
+
+class BufferInterfaceNotFound(Exception):
+    pass
+
+
+class BufferView(object):
+    """Abstract base class for buffers."""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        return ''.join(self._copy_buffer())
+
+    def getbytes(self, start, size):
+        """Return `size` bytes starting at byte offset `start`.
+
+        This is a low-level operation, it is up to the caller to ensure that
+        the data requested actually correspond to items accessible from the
+        BufferView.
+        Note that `start` may be negative, e.g. if the buffer is reversed.
+        """
+        raise NotImplementedError
+
+    def setbytes(self, start, string):
+        raise NotImplementedError
+
+    def get_raw_address(self):
+        raise ValueError("no raw buffer")
+
+    def as_readbuf(self):
+        # Inefficient. May be overridden.
+        return StringBuffer(self.as_str())
+
+    def as_writebuf(self):
+        """Return a writable Buffer sharing the same data as `self`."""
+        raise BufferInterfaceNotFound
+
+    def getformat(self):
+        raise NotImplementedError
+
+    def getitemsize(self):
+        raise NotImplementedError
+
+    def getndim(self):
+        raise NotImplementedError
+
+    def getshape(self):
+        raise NotImplementedError
+
+    def getstrides(self):
+        raise NotImplementedError
+
+    def releasebuffer(self):
+        pass
+
+    def value_from_bytes(self, space, s):
+        from pypy.module.struct.formatiterator import UnpackFormatIterator
+        buf = StringBuffer(s)
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(self.getformat())
+        return fmtiter.result_w[0]
+
+    def _copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.getbytes(0, itemsize)]
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim + 1, data, off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        bytesize = self.getlength()
+        copiedbytes = 0
+        for i in range(step):
+            bytes = self.getbytes(off, itemsize)
+            data.append(bytes)
+            copiedbytes += len(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if copiedbytes >= bytesize:
+                break
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        shape = self.getshape()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        # TODO suboffsets?
+        strides = self.getstrides()
+        return strides[dim] * index
+
+    def w_getitem(self, space, idx):
+        offset = self.get_offset(space, 0, idx)
+        itemsize = self.getitemsize()
+        # TODO: this probably isn't very fast
+        data = self.getbytes(offset, itemsize)
+        return space.newbytes(data)
+
+    def new_slice(self, start, step, slicelength):
+        return BufferSlice(self, start, step, slicelength)
+
+    def w_tolist(self, space):
+        dim = self.getndim()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            n = self.getshape()[0]
+            values_w = [space.ord(self.w_getitem(space, i)) for i in range(n)]
+            return space.newlist(values_w)
+        else:
+            return self._tolist_rec(space, 0, 0)
+
+    def _tolist_rec(self, space, start, idim):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim + 1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            values_w = [
+                self.value_from_bytes(space, self.getbytes(pos, itemsize))
+                for pos in range(start, start + bytecount, stride)]
+            return space.newlist(values_w)
+
+        items = [None] * dimshape
+        for i in range(dimshape):
+            item = self._tolist_rec(space, start, idim + 1)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+    def wrap(self, space):
+        return space.newmemoryview(self)
+
+
+class SimpleView(BufferView):
+    _attrs_ = ['readonly', 'data']
+    _immutable_ = True
+
+    def __init__(self, data):
+        self.data = data
+        self.readonly = self.data.readonly
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def as_str(self):
+        return self.data.as_str()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def setbytes(self, offset, s):
+        self.data.setslice(offset, s)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_readbuf(self):
+        return self.data
+
+    def as_writebuf(self):
+        assert not self.data.readonly
+        return self.data
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        assert dim == 0
+        nitems = self.getlength()
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        return index
+
+    def w_getitem(self, space, idx):
+        idx = self.get_offset(space, 0, idx)
+        ch = self.data[idx]
+        return space.newbytes(ch)
+
+    def new_slice(self, start, step, slicelength):
+        if step == 1:
+            return SimpleView(SubBuffer(self.data, start, slicelength))
+        else:
+            return BufferSlice(self, start, step, slicelength)
+
+
+class BufferSlice(BufferView):
+    _immutable_ = True
+    _attrs_ = ['parent', 'readonly', 'shape', 'strides', 'start', 'step']
+
+    def __init__(self, parent, start, step, length):
+        self.parent = parent
+        self.readonly = self.parent.readonly
+        self.strides = parent.getstrides()[:]
+        self.start = start
+        self.step = step
+        self.strides[0] *= step
+        self.shape = parent.getshape()[:]
+        self.shape[0] = length
+
+    def getlength(self):
+        return self.shape[0] * self.getitemsize()
+
+    def getbytes(self, start, size):
+        offset = self.start * self.parent.getstrides()[0]
+        return self.parent.getbytes(offset + start, size)
+
+    def setbytes(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        offset = self.start * self.parent.getstrides()[0]
+        self.parent.setbytes(offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        offset = self.start * self.parent.getstrides()[0]
+        return rffi.ptradd(self.parent.get_raw_address(), offset)
+
+    def getformat(self):
+        return self.parent.getformat()
+
+    def getitemsize(self):
+        return self.parent.getitemsize()
+
+    def getndim(self):
+        return self.parent.getndim()
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides
+
+    def parent_index(self, idx):
+        return self.start + self.step * idx
+
+    def w_getitem(self, space, idx):
+        return self.parent.w_getitem(space, self.parent_index(idx))
+
+    def new_slice(self, start, step, slicelength):
+        real_start = start + self.start
+        real_step = self.step * step
+        return BufferSlice(self.parent, real_start, real_step, slicelength)
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,5 +1,6 @@
 class AppTestCompile:
     def test_simple(self):
+        import sys
         co = compile('1+2', '?', 'eval')
         assert eval(co) == 3
         co = compile(buffer('1+2'), '?', 'eval')
@@ -8,8 +9,10 @@
         assert str(exc.value) == "compile() expected string without null bytes"
         exc = raises(TypeError, compile, unichr(0), '?', 'eval')
         assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
+
+        if '__pypy__' in sys.modules:
+            co = compile(memoryview('1+2'), '?', 'eval')
+            assert eval(co) == 3
         compile("from __future__ import with_statement", "<test>", "exec")
         raises(SyntaxError, compile, '-', '?', 'eval')
         raises(ValueError, compile, '"\\xt"', '?', 'eval')
@@ -50,7 +53,8 @@
         co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -4,6 +4,7 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 from pypy.objspace.std.bufferobject import W_Buffer
+from pypy.interpreter.buffer import SimpleView
 
 from rpython.rlib.buffer import RawBuffer
 from rpython.rtyper.annlowlevel import llstr
@@ -60,7 +61,7 @@
         if space.isinstance_w(w_other, space.w_unicode):
             return space.w_NotImplemented
         try:
-            other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+            other_buf = space.readbuf_w(w_other)
         except OperationError as e:
             if e.async(space):
                 raise
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,24 +105,10 @@
 # ____________________________________________________________
 
 def _fetch_as_read_buffer(space, w_x):
-    # xxx do we really need to implement the same mess as in CPython 2.7
-    # w.r.t. buffers and memoryviews??
-    try:
-        buf = space.readbuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_SIMPLE)
-    return buf
+    return space.readbuf_w(w_x)
 
 def _fetch_as_write_buffer(space, w_x):
-    try:
-        buf = space.writebuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_WRITABLE)
-    return buf
+    return space.writebuf_w(w_x)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def from_buffer(space, w_ctype, w_x):
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -292,8 +292,8 @@
             assert bytes2.decode("unicode_internal") == u"\U00010098"
         assert bytes.decode("unicode_internal") == u"a"
         assert _codecs.unicode_internal_decode(array.array('c', bytes))[0] == u"a"
-        exc = raises(TypeError, _codecs.unicode_internal_decode, memoryview(bytes))
-        assert str(exc.value) == "expected a readable buffer object"
+        if '__pypy__' in sys.modules:
+            assert _codecs.unicode_internal_decode(memoryview(bytes))[0] == u"a"
 
     def test_raw_unicode_escape(self):
         assert unicode("\u0663", "raw-unicode-escape") == u"\u0663"
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -1,12 +1,15 @@
 from __future__ import with_statement
 
+from rpython.rlib.signature import signature
+from rpython.rlib import types
+
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rgc import (
-    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import SimpleView
+
+from rpython.rlib.buffer import ByteBuffer, SubBuffer
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib import rposix
@@ -16,7 +19,6 @@
     check_readable_w, check_writable_w, check_seekable_w)
 from pypy.module._io.interp_io import W_BlockingIOError
 from rpython.rlib import rthread
-from rpython.rtyper.lltypesystem import rffi
 
 STATE_ZERO, STATE_OK, STATE_DETACHED = range(3)
 
@@ -88,12 +90,16 @@
         self._unsupportedoperation(space, "detach")
 
     def readinto_w(self, space, w_buffer):
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        return self._readinto(space, w_buffer, "read")
+
+    def _readinto(self, space, w_buffer, methodname):
+        rwbuffer = space.writebuf_w(w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, "read", space.newint(length))
+        w_data = space.call_method(self, methodname, space.newint(length))
 
         if not space.isinstance_w(w_data, space.w_bytes):
-            raise oefmt(space.w_TypeError, "read() should return bytes")
+            raise oefmt(space.w_TypeError, "%s() should return bytes",
+                        methodname)
         data = space.bytes_w(w_data)
         rwbuffer.setslice(0, data)
         return space.newint(len(data))
@@ -108,25 +114,6 @@
     readinto = interp2app(W_BufferedIOBase.readinto_w),
 )
 
-class RawBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, buf, start, length):
-        self.buf = buf
-        self.start = start
-        self.length = length
-        self.readonly = False
-
-    def getlength(self):
-        return self.length
-
-    def setitem(self, index, char):
-        self.buf[self.start + index] = char
-
-    def get_raw_address(self):
-        ptr = nonmoving_raw_ptr_for_resizable_list(self.buf)
-        return rffi.ptradd(ptr, self.start)
-
 class BufferedMixin:
     _mixin_ = True
 
@@ -165,8 +152,7 @@
             raise oefmt(space.w_ValueError,
                         "buffer size must be strictly positive")
 
-        self.buffer = resizable_list_supporting_raw_ptr(['\0'] *
-                                                        self.buffer_size)
+        self.buffer = ByteBuffer(self.buffer_size)
 
         self.lock = TryLock(space)
 
@@ -238,6 +224,7 @@
 
     # ______________________________________________
 
+    @signature(types.any(), returns=types.int())
     def _readahead(self):
         if self.readable and self.read_end != -1:
             available = self.read_end - self.pos
@@ -278,7 +265,7 @@
                 else:
                     offset = pos
                 if -self.pos <= offset <= available:
-                    newpos = self.pos + offset
+                    newpos = self.pos + int(offset)
                     assert newpos >= 0
                     self.pos = newpos
                     return space.newint(current - available + offset)
@@ -374,11 +361,7 @@
         return written
 
     def _raw_write(self, space, start, end):
-        # XXX inefficient
-        l = []
-        for i in range(start, end):
-            l.append(self.buffer[i])
-        return self._write(space, ''.join(l))
+        return self._write(space, self.buffer[start:end])
 
     def detach_w(self, space):
         self._check_init(space)
@@ -428,6 +411,7 @@
     @unwrap_spec(size=int)
     def peek_w(self, space, size=0):
         self._check_init(space)
+        self._check_closed(space, "peek of closed file")
         with self.lock:
             if self.writable:
                 self._flush_and_rewind_unlocked(space)
@@ -439,7 +423,7 @@
             # buffer.
             have = self._readahead()
             if have > 0:
-                data = ''.join(self.buffer[self.pos:self.pos+have])
+                data = self.buffer[self.pos:self.pos+have]
                 return space.newbytes(data)
 
             # Fill the buffer from the raw stream, and copy it to the result
@@ -449,7 +433,7 @@
             except BlockingIOError:
                 size = 0
             self.pos = 0
-            data = ''.join(self.buffer[:size])
+            data = self.buffer[0:size]
             return space.newbytes(data)
 
     @unwrap_spec(size=int)
@@ -486,7 +470,7 @@
             if size > have:
                 size = have
             endpos = self.pos + size
-            data = ''.join(self.buffer[self.pos:endpos])
+            data = self.buffer[self.pos:endpos]
             self.pos = endpos
             return space.newbytes(data)
 
@@ -498,7 +482,7 @@
         current_size = self._readahead()
         data = None
         if current_size:
-            data = ''.join(self.buffer[self.pos:self.pos + current_size])
+            data = self.buffer[self.pos:self.pos + current_size]
             builder.append(data)
             self.pos += current_size
         # We're going past the buffer's bounds, flush it
@@ -524,11 +508,13 @@
         return space.newbytes(builder.build())
 
     def _raw_read(self, space, buffer, start, length):
+        assert buffer is not None
         length = intmask(length)
-        w_buf = space.newbuffer(RawBuffer(buffer, start, length))
+        start = intmask(start)
+        w_view = SimpleView(SubBuffer(buffer, start, length)).wrap(space)
         while True:
             try:
-                w_size = space.call_method(self.w_raw, "readinto", w_buf)
+                w_size = space.call_method(self.w_raw, "readinto", w_view)
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue  # try again
@@ -565,12 +551,12 @@
         if n <= current_size:
             return self._read_fast(n)
 
-        result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n)
+        result_buffer = ByteBuffer(n)
         remaining = n
         written = 0
         if current_size:
-            for i in range(current_size):
-                result_buffer[written + i] = self.buffer[self.pos + i]
+            result_buffer.setslice(
+                written, self.buffer[self.pos:self.pos + current_size])
             remaining -= current_size
             written += current_size
             self.pos += current_size
@@ -592,7 +578,7 @@
                     return None
                 size = 0
             if size == 0:
-                return ''.join(result_buffer[:written])
+                return result_buffer[0:written]
             remaining -= size
             written += size
 
@@ -614,14 +600,13 @@
             if remaining > 0:
                 if size > remaining:
                     size = remaining
-                for i in range(size):
-                    result_buffer[written + i] = self.buffer[self.pos + i]
+                result_buffer.setslice(
+                    written, self.buffer[self.pos:self.pos + size])
                 self.pos += size
-
                 written += size
                 remaining -= size
 
-        return ''.join(result_buffer[:written])
+        return result_buffer[0:written]
 
     def _read_fast(self, n):
         """Read n bytes from the buffer if it can, otherwise return None.
@@ -629,7 +614,7 @@
         current_size = self._readahead()
         if n <= current_size:
             endpos = self.pos + n
-            res = ''.join(self.buffer[self.pos:endpos])
+            res = self.buffer[self.pos:endpos]
             self.pos = endpos
             return res
         return None
@@ -652,11 +637,11 @@
         else:
             pos = -1
         if pos >= 0:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1]))
+            w_res = space.newbytes(self.buffer[self.pos:pos+1])
             self.pos = pos + 1
             return w_res
         if have == limit:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have]))
+            w_res = space.newbytes(self.buffer[self.pos:self.pos+have])
             self.pos += have
             return w_res
 
@@ -665,7 +650,7 @@
             # Now we try to get some more from the raw stream
             chunks = []
             if have > 0:
-                chunks.extend(self.buffer[self.pos:self.pos+have])
+                chunks.append(self.buffer[self.pos:self.pos+have])
                 written += have
                 self.pos += have
                 if limit >= 0:
@@ -683,13 +668,14 @@
                 pos = 0
                 found = False
                 while pos < have:
-                    c = self.buffer[pos]
+                    # 'buffer.data[]' instead of 'buffer[]' because RPython...
+                    c = self.buffer.data[pos]
                     pos += 1
                     if c == '\n':
                         self.pos = pos
                         found = True
                         break
-                chunks.extend(self.buffer[0:pos])
+                chunks.append(self.buffer[0:pos])
                 if found:
                     break
                 if have == limit:
@@ -716,7 +702,6 @@
         size = len(data)
 
         with self.lock:
-
             if (not (self.readable and self.read_end != -1) and
                 not (self.writable and self.write_end != -1)):
                 self.pos = 0
@@ -746,7 +731,8 @@
                     self._reader_reset_buf()
                 # Make some place by shifting the buffer
                 for i in range(self.write_pos, self.write_end):
-                    self.buffer[i - self.write_pos] = self.buffer[i]
+                    # XXX: messing with buffer internals
+                    self.buffer.data[i - self.write_pos] = self.buffer.data[i]
                 self.write_end -= self.write_pos
                 self.raw_pos -= self.write_pos
                 newpos = self.pos - self.write_pos
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -34,17 +34,17 @@
         size = convert_size(space, w_size)
         return space.newbytes(self.read(size))
 
+    def read1_w(self, space, w_size):
+        return self.read_w(space, w_size)
+
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
         return space.newbytes(self.readline(limit))
 
-    def read1_w(self, space, w_size):
-        return self.read_w(space, w_size)
-
     def readinto_w(self, space, w_buffer):
         self._check_closed(space)
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        rwbuffer = space.writebuf_w(w_buffer)
         size = rwbuffer.getlength()
 
         output = self.read(size)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,5 +1,5 @@
+from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.buffer import RawBuffer
-from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -1,5 +1,6 @@
 import sys
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
@@ -371,7 +372,7 @@
         self._ll_buffer = self.ll_buffer
 
     def buffer_w(self, space, flags):
-        return RawFFIBuffer(self)
+        return SimpleView(RawFFIBuffer(self))
 
     def readbuf_w(self, space):
         return RawFFIBuffer(self)
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -41,15 +41,15 @@
     assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
     view = py_obj.c_view
-    ndim = w_obj.buf.getndim()
+    ndim = w_obj.getndim()
     if ndim >= Py_MAX_NDIMS:
         # XXX warn?
         return
-    fill_Py_buffer(space, w_obj.buf, view)
+    fill_Py_buffer(space, w_obj.view, view)
     try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
         view.c_obj = make_ref(space, w_userdata)
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
         view.c_obj = make_ref(space, w_s)
@@ -95,7 +95,6 @@
     mem_obj.c_view.c_obj = rffi.cast(PyObject, 0)
     _dealloc(space, py_obj)
 
-
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -19,6 +19,7 @@
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
 from rpython.rlib.buffer import RawBuffer
@@ -322,7 +323,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(RawBuffer):
+class CPyBuffer(BufferView):
     # Similar to Py_buffer
     _immutable_ = True
 
@@ -333,9 +334,12 @@
         self.space = space
         self.ptr = ptr
         self.size = size
-        self.w_obj = w_obj # kept alive
+        self.w_obj = w_obj  # kept alive
         self.pyobj = as_pyobj(space, w_obj)
         self.format = format
+        self.ndim = ndim
+        self.itemsize = itemsize
+
         if not shape:
             self.shape = [size]
         else:
@@ -344,8 +348,6 @@
             self.strides = [1]
         else:
             self.strides = strides
-        self.ndim = ndim
-        self.itemsize = itemsize
         self.readonly = readonly
         self.needs_decref = needs_decref
         self.releasebufferproc = releasebufferproc
@@ -378,8 +380,20 @@
     def getlength(self):
         return self.size
 
-    def getitem(self, index):
-        return self.ptr[index]
+    def getbytes(self, start, size):
+        return ''.join([self.ptr[i] for i in range(start, start + size)])
+
+    def setbytes(self, start, string):
+        # absolutely no safety checks, what could go wrong?
+        for i in range(len(string)):
+            self.ptr[start + i] = string[i]
+
+    def as_readbuf(self):
+        return CBuffer(self)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return CBuffer(self)
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
@@ -399,10 +413,6 @@
     def getndim(self):
         return self.ndim
 
-    def setitem(self, index, char):
-        # absolutely no safety checks, what could go wrong?
-        self.ptr[index] = char
-
 class FQ(rgc.FinalizerQueue):
     Class = CPyBuffer
     def finalizer_trigger(self):
@@ -414,6 +424,37 @@
 
 fq = FQ()
 
+
+class CBuffer(RawBuffer):
+    _immutable_ = True
+    def __init__(self, view):
+        self.view = view
+        self.readonly = view.readonly
+
+    def getlength(self):
+        return self.view.getlength()
+
+    def getitem(self, index):
+        return self.view.ptr[index]
+
+    def getslice(self, start, stop, step, size):
+        assert step == 1
+        assert stop - start == size
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), start)
+        return rffi.charpsize2str(ptr, size)
+
+    def setitem(self, index, char):
+        self.view.ptr[index] = char
+
+    def setslice(self, index, s):
+        assert s is not None
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), index)
+        rffi.str2chararray(s, ptr, len(s))
+
+    def get_raw_address(self):
+        return cts.cast('char *', self.view.ptr)
+
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
@@ -427,10 +468,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self,
+        view = CPyBuffer(space, ptr[0], size, w_self,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getwritebuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -445,10 +486,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+        view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
@@ -490,7 +531,7 @@
                             needs_decref=True,
                             releasebufferproc = rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return buf.wrap(space)
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -667,6 +708,7 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         slot_func = slot_tp_getattro
+
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -744,24 +786,24 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         args = Arguments(space, [space.newint(flags)])
         w_obj = space.call_args(space.get(buff_fn, w_self), args)
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
@@ -771,23 +813,23 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         w_obj = w_self
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,6 +344,7 @@
 #endif
     if (m == NULL)
         INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -111,7 +111,7 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyLong_FromLong(-1);
+                    return NULL;
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
                 return PyLong_FromLong(view->len / view->itemsize);
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -34,7 +34,7 @@
 cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct)
 
 @bootstrap_function
-def init_stringobject(space):
+def init_tupleobject(space):
     "Type description of PyTupleObject"
     make_typedescr(space.w_tuple.layout.typedef,
                    basestruct=PyTupleObject.TO,
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -920,90 +920,42 @@
 class W_GroupBy(W_Root):
     def __init__(self, space, w_iterable, w_fun):
         self.space = space
-        self.w_iterable = self.space.iter(w_iterable)
-        if space.is_none(w_fun):
-            self.w_fun = None
-        else:
-            self.w_fun = w_fun
-        self.index = 0
-        self.lookahead = False
-        self.exhausted = False
-        self.started = False
-        # new_group - new group not started yet, next should not skip any items
-        self.new_group = True
-        self.w_lookahead = self.space.w_None
-        self.w_key = self.space.w_None
+        self.w_iterator = self.space.iter(w_iterable)
+        if w_fun is None:
+            w_fun = space.w_None
+        self.w_keyfunc = w_fun
+        self.w_tgtkey = None
+        self.w_currkey = None
+        self.w_currvalue = None
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        self._skip_to_next_iteration_group()
+        w_key = self.w_tgtkey = self.w_currkey
+        w_grouper = W_GroupByIterator(self, w_key)
+        return self.space.newtuple([w_key, w_grouper])
 
-        if not self.new_group:
-            self._consume_unwanted_input()
+    def _skip_to_next_iteration_group(self):
+        space = self.space
+        while True:
+            if self.w_currkey is None:
+                pass
+            elif self.w_tgtkey is None:
+                break
+            else:
+                if not space.eq_w(self.w_tgtkey, self.w_currkey):
+                    break
 
-        if not self.started:
-            self.started = True
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                raise
+            w_newvalue = space.next(self.w_iterator)
+            if space.is_w(self.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
             else:
-                self.w_lookahead = w_obj
-                if self.w_fun is None:
-                    self.w_key = w_obj
-                else:
-                    self.w_key = self.space.call_function(self.w_fun, w_obj)
-                self.lookahead = True
+                w_newkey = space.call_function(self.w_keyfunc, w_newvalue)
 
-        self.new_group = False
-        w_iterator = W_GroupByIterator(self.space, self.index, self)
-        return self.space.newtuple([self.w_key, w_iterator])
-
-    def _consume_unwanted_input(self):
-        # Consume unwanted input until we reach the next group
-        try:
-            while True:
-                self.group_next(self.index)
-        except StopIteration:
-            pass
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
-    def group_next(self, group_index):
-        if group_index < self.index:
-            raise StopIteration
-        else:
-            if self.lookahead:
-                self.lookahead = False
-                return self.w_lookahead
-
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                    raise StopIteration
-                else:
-                    raise
-            else:
-                if self.w_fun is None:
-                    w_new_key = w_obj
-                else:
-                    w_new_key = self.space.call_function(self.w_fun, w_obj)
-                if self.space.eq_w(self.w_key, w_new_key):
-                    return w_obj
-                else:
-                    self.index += 1
-                    self.w_lookahead = w_obj
-                    self.w_key = w_new_key
-                    self.lookahead = True
-                    self.new_group = True #new group
-                    raise StopIteration
+            self.w_currkey = w_newkey
+            self.w_currvalue = w_newvalue
 
 def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
     r = space.allocate_instance(W_GroupBy, w_subtype)
@@ -1036,26 +988,34 @@
 
 
 class W_GroupByIterator(W_Root):
-    def __init__(self, space, index, groupby):
-        self.space = space
-        self.index = index
+    def __init__(self, groupby, w_tgtkey):
         self.groupby = groupby
-        self.exhausted = False
+        self.w_tgtkey = w_tgtkey
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        groupby = self.groupby
+        space = groupby.space
+        if groupby.w_currvalue is None:
+            w_newvalue = space.next(groupby.w_iterator)
+            if space.is_w(groupby.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
+            else:
+                w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
+            #assert groupby.w_currvalue is None
+            # ^^^ check disabled, see http://bugs.python.org/issue30347
+            groupby.w_currkey = w_newkey
+            groupby.w_currvalue = w_newvalue
 
-        try:
-            w_obj = self.groupby.group_next(self.index)
-        except StopIteration:
-            self.exhausted = True
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        else:
-            return w_obj
+        assert groupby.w_currkey is not None
+        if not space.eq_w(self.w_tgtkey, groupby.w_currkey):
+            raise OperationError(space.w_StopIteration, space.w_None)
+        w_result = groupby.w_currvalue
+        groupby.w_currvalue = None
+        groupby.w_currkey = None
+        return w_result
 
 W_GroupByIterator.typedef = TypeDef(
         'itertools._groupby',
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -634,6 +634,17 @@
         it = itertools.groupby([0], 1)
         raises(TypeError, it.next)
 
+    def test_groupby_question_43905804(self):
+        # http://stackoverflow.com/questions/43905804/
+        import itertools
+
+        inputs = ((x > 5, x) for x in range(10))
+        (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+        a = list(a)
+        b = list(b)
+        assert a == []
+        assert b == [(True, 9)]
+
     def test_iterables(self):
         import itertools
     
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
@@ -21,7 +22,7 @@
 
 TimSort = make_timsort_class()
 class StrideSort(TimSort):
-    ''' 
+    '''
     argsort (return the indices to sort) a list of strides
     '''
     def __init__(self, rangelist, strides, order):
@@ -380,14 +381,14 @@
 
     def get_buffer(self, space, flags):
         errtype = space.w_ValueError # should be BufferError, numpy does this instead
-        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_C_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not C-contiguous")
-        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
@@ -397,7 +398,7 @@
             not self.flags & NPY.ARRAY_WRITEABLE):
            raise oefmt(errtype, "buffer source array is read-only")
         readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
-        return ArrayBuffer(self, readonly)
+        return ArrayView(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
         # copy the general pattern of the strides
@@ -527,7 +528,7 @@
         try:
             length = support.product_check(shape)
             self.size = ovfcheck(length * dtype.elsize)
-        except OverflowError: 
+        except OverflowError:
             raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
         if storage == lltype.nullptr(RAW_STORAGE):
             if dtype.num == NPY.OBJECT:
@@ -702,9 +703,8 @@
         free_raw_storage(self.storage)
 
 
-class ArrayBuffer(RawBuffer):
+class ArrayData(RawBuffer):
     _immutable_ = True
-
     def __init__(self, impl, readonly):
         self.impl = impl
         self.readonly = readonly
@@ -725,6 +725,28 @@
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
 
+
+class ArrayView(BufferView):
+    _immutable_ = True
+
+    def __init__(self, impl, readonly):
+        self.impl = impl
+        self.readonly = readonly
+        self.data = ArrayData(impl, readonly)
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def as_readbuf(self):
+        return ArrayData(self.impl, readonly=True)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return ArrayData(self.impl, readonly=False)
+
     def getformat(self):
         sb = StringBuilder()
         self.impl.dtype.getformat(sb)
@@ -742,4 +764,5 @@
     def getstrides(self):
         return self.impl.strides
 
-
+    def get_raw_address(self):
+        return self.data.get_raw_address()
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -91,7 +91,7 @@
             w_base = w_object
             if read_only:
                 w_base = None
-            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data,
                                 dtype, w_base=w_base, strides=strides,
                                 start=offset), read_only
         if w_data is None:
@@ -104,11 +104,11 @@
         #print 'create view from shape',shape,'dtype',dtype,'data',data
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
-                   "__array_interface__ strides not fully supported yet") 
+                   "__array_interface__ strides not fully supported yet")
         arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
-        
+
     except OperationError as e:
         if e.match(space, space.w_AttributeError):
             return None, False
@@ -120,7 +120,7 @@
         return descr
     msg = "invalid PEP 3118 format string: '%s'" % c_format
     space.warn(space.newtext(msg), space.w_RuntimeWarning)
-    return None 
+    return None
 
 def _array_from_buffer_3118(space, w_object, dtype):
     try:
@@ -139,12 +139,12 @@
             raise oefmt(space.w_NotImplementedError,
                 "creating an array from a memoryview while specifying dtype "
                 "not supported")
-        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))): 
+        if descr.elsize != space.int_w(space.getattr(w_buf, space.newbytes('itemsize'))):
             msg = ("Item size computed from the PEP 3118 buffer format "
                   "string does not match the actual item size.")
             space.warn(space.newtext(msg), space.w_RuntimeWarning)
             return w_object
-        dtype = descr 
+        dtype = descr
     elif not dtype:
         dtype = descriptor.get_dtype_cache(space).w_stringdtype
         dtype.elsize = space.int_w(space.getattr(w_buf, space.newbytes('itemsize')))
@@ -181,7 +181,7 @@
             raise e
     writable = not space.bool_w(space.getattr(w_buf, space.newbytes('readonly')))
     w_ret = W_NDimArray.from_shape_and_storage(space, shape, w_data,
-               storage_bytes=buflen, dtype=dtype, w_base=w_object, 
+               storage_bytes=buflen, dtype=dtype, w_base=w_object,
                writable=writable, strides=strides)
     if w_ret:
         return w_ret
@@ -212,7 +212,7 @@
     if not isinstance(w_object, W_NDimArray):
         w_array = try_array_method(space, w_object, w_dtype)
         if w_array is None:
-            if (    not space.isinstance_w(w_object, space.w_bytes) and 
+            if (    not space.isinstance_w(w_object, space.w_bytes) and
                     not space.isinstance_w(w_object, space.w_unicode) and
                     not isinstance(w_object, W_GenericBox)):
                 # use buffer interface
@@ -551,7 +551,7 @@
     except OperationError as e:
         if not e.match(space, space.w_TypeError):
             raise
-        w_buffer = space.call_method(w_buffer, '__buffer__', 
+        w_buffer = space.call_method(w_buffer, '__buffer__',
                                     space.newint(space.BUF_FULL_RO))
         buf = _getbuffer(space, w_buffer)
 
diff --git a/pypy/module/micronumpy/ndarray.py b/pypy/module/micronumpy/ndarray.py
--- a/pypy/module/micronumpy/ndarray.py
+++ b/pypy/module/micronumpy/ndarray.py
@@ -3,6 +3,7 @@
     WrappedDefault
 from pypy.interpreter.typedef import TypeDef, GetSetProperty, \
     make_weakref_descr
+from pypy.interpreter.buffer import SimpleView
 from rpython.rlib import jit
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import RAW_STORAGE_PTR
@@ -807,16 +808,17 @@
         return self.implementation.get_buffer(space, flags)
 
     def readbuf_w(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL_RO)
+        return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_readbuf()
 
     def writebuf_w(self, space):
-        return self.implementation.get_buffer(space, space.BUF_FULL)
+        return self.implementation.get_buffer(space, space.BUF_FULL).as_writebuf()
 
     def charbuf_w(self, space):
         return self.implementation.get_buffer(space, space.BUF_FULL_RO).as_str()
 
     def descr_get_data(self, space):
-        return space.newbuffer(self.implementation.get_buffer(space, space.BUF_FULL))
+        return space.newbuffer(
+            self.implementation.get_buffer(space, space.BUF_FULL).as_writebuf())
 
     @unwrap_spec(offset=int, axis1=int, axis2=int)
     def descr_diagonal(self, space, offset=0, axis1=0, axis2=1):
diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py
--- a/pypy/module/micronumpy/test/test_ndarray.py
+++ b/pypy/module/micronumpy/test/test_ndarray.py
@@ -3611,8 +3611,6 @@
         import numpy as np
         exc = raises(AttributeError, np.frombuffer, None)
         assert str(exc.value) == "'NoneType' object has no attribute '__buffer__'"
-        exc = raises(AttributeError, np.frombuffer, memoryview(self.data))
-        assert str(exc.value) == "'memoryview' object has no attribute '__buffer__'"
         exc = raises(ValueError, np.frombuffer, self.data, 'S0')
         assert str(exc.value) == "itemsize cannot be zero in type"
         exc = raises(ValueError, np.frombuffer, self.data, offset=-1)
@@ -3684,7 +3682,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
         dt1 = np.dtype(
              [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')],
@@ -3695,7 +3693,7 @@
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}'
         else:
             assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx at i:c:}'
- 
+
 
     def test_fromstring(self):
         import sys
diff --git a/pypy/module/micronumpy/types.py b/pypy/module/micronumpy/types.py
--- a/pypy/module/micronumpy/types.py
+++ b/pypy/module/micronumpy/types.py
@@ -454,8 +454,8 @@
             return Float64(self.space).box(self.unbox(v))
         # numpy 1.10 compatibility
         raise oefmt(self.space.w_TypeError, "ufunc casting failure")
-            
-            
+
+
 
 class Integer(Primitive):
     _mixin_ = True
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -121,16 +121,17 @@
 class W_Struct(W_Root):
     _immutable_fields_ = ["format", "size"]
 
-    def __init__(self, space, format):
+    format = ""
+    size = -1
+
+    def descr__new__(space, w_subtype, __args__):
+        return space.allocate_instance(W_Struct, w_subtype)
+
+    @unwrap_spec(format='text')
+    def descr__init__(self, space, format):
         self.format = format
         self.size = _calcsize(space, format)
 
-    @unwrap_spec(format='text')
-    def descr__new__(space, w_subtype, format):
-        self = space.allocate_instance(W_Struct, w_subtype)
-        W_Struct.__init__(self, space, format)
-        return self
-
     def descr_pack(self, space, args_w):
         return pack(space, jit.promote_string(self.format), args_w)
 
@@ -147,6 +148,7 @@
 
 W_Struct.typedef = TypeDef("Struct",
     __new__=interp2app(W_Struct.descr__new__.im_func),
+    __init__=interp2app(W_Struct.descr__init__),
     format=interp_attrproperty("format", cls=W_Struct, wrapfn="newbytes"),
     size=interp_attrproperty("size", cls=W_Struct, wrapfn="newint"),
 
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -382,6 +382,7 @@
         raises(self.struct.error, self.struct.unpack, "i", b)
 
     def test_pack_unpack_buffer(self):
+        import sys
         import array
         b = array.array('c', '\x00' * 19)
         sz = self.struct.calcsize("ii")
@@ -391,9 +392,11 @@
                                       self.struct.pack("ii", 17, 42) +
                                       '\x00' * (19-sz-2))
         exc = raises(TypeError, self.struct.pack_into, "ii", buffer(b), 0, 17, 42)
-        assert str(exc.value) == "must be read-write buffer, not buffer"
+        if '__pypy__' in sys.modules:
+            assert str(exc.value) == "must be read-write buffer, not buffer"
         exc = raises(TypeError, self.struct.pack_into, "ii", 'test', 0, 17, 42)
-        assert str(exc.value) == "must be read-write buffer, not str"
+        if '__pypy__' in sys.modules:
+            assert str(exc.value) == "must be read-write buffer, not str"
         exc = raises(self.struct.error, self.struct.pack_into, "ii", b[0:1], 0, 17, 42)
         assert str(exc.value) == "pack_into requires a buffer of at least 8 bytes"
 
@@ -429,6 +432,14 @@
         assert s.unpack(s.pack(42)) == (42,)
         assert s.unpack_from(memoryview(s.pack(42))) == (42,)
 
+    def test_struct_subclass(self):
+        class S(self.struct.Struct):
+            def __init__(self):
+                assert self.size == -1
+                super(S, self).__init__('c')
+                assert self.size == 1
+        assert S().unpack('a') == ('a',)
+
     def test_overflow(self):
         raises(self.struct.error, self.struct.pack, 'i', 1<<65)
 
diff --git a/pypy/objspace/fake/objspace.py b/pypy/objspace/fake/objspace.py
--- a/pypy/objspace/fake/objspace.py
+++ b/pypy/objspace/fake/objspace.py
@@ -1,9 +1,4 @@
 from rpython.annotator.model import SomeInstance, s_None
-from pypy.interpreter import argument, gateway
-from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
-from pypy.interpreter.typedef import TypeDef, GetSetProperty
-from pypy.objspace.std.sliceobject import W_SliceObject
-from rpython.rlib.buffer import StringBuffer
 from rpython.rlib.objectmodel import (instantiate, we_are_translated, specialize,
     not_rpython)
 from rpython.rlib.nonconst import NonConstant
@@ -14,6 +9,13 @@
 from rpython.tool.sourcetools import compile2, func_with_new_name
 from rpython.translator.translator import TranslationContext
 
+from pypy.tool.option import make_config
+from pypy.interpreter import argument, gateway
+from pypy.interpreter.baseobjspace import W_Root, ObjSpace, SpaceCache
+from pypy.interpreter.buffer import StringBuffer, SimpleView
+from pypy.interpreter.typedef import TypeDef, GetSetProperty
+from pypy.objspace.std.sliceobject import W_SliceObject
+
 
 class W_MyObject(W_Root):
     typedef = None
@@ -41,7 +43,7 @@
         is_root(w_subtype)
 
     def buffer_w(self, space, flags):
-        return StringBuffer("foobar")
+        return SimpleView(StringBuffer("foobar"))
 
     def str_w(self, space):
         return NonConstant("foobar")
@@ -123,7 +125,7 @@
 
 BUILTIN_TYPES = ['int', 'str', 'float', 'long', 'tuple', 'list', 'dict',
                  'unicode', 'complex', 'slice', 'bool', 'basestring', 'object',
-                 'bytearray', 'buffer', 'set', 'frozenset']
+                 'set', 'frozenset', 'bytearray', 'buffer', 'memoryview']
 
 INTERP_TYPES = ['function', 'builtin_function', 'module', 'getset_descriptor',
                 'instance', 'classobj']
@@ -197,7 +199,7 @@
     def newseqiter(self, x):
         return w_some_obj()
 
-    def newbuffer(self, x):
+    def newmemoryview(self, x):
         return w_some_obj()
 
     @not_rpython
diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -5,6 +5,7 @@
 from rpython.rlib.objectmodel import compute_hash
 
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView, BufferInterfaceNotFound
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
@@ -19,7 +20,7 @@
 
     def buffer_w(self, space, flags):
         space.check_buf_flags(flags, self.buf.readonly)
-        return self.buf
+        return SimpleView(self.buf)
 
     def readbuf_w(self, space):
         return self.buf
@@ -39,7 +40,10 @@
     @staticmethod
     @unwrap_spec(offset=int, size=int)
     def descr_new_buffer(space, w_subtype, w_object, offset=0, size=-1):
-        buf = space.readbuf_w(w_object)
+        try:
+            buf = w_object.readbuf_w(space)
+        except BufferInterfaceNotFound:
+            raise oefmt(space.w_TypeError, "expected a readable buffer object")
         if offset == 0 and size == -1:
             return W_Buffer(buf)
         # handle buffer slices
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -2,57 +2,70 @@
 
 from rpython.rlib.objectmodel import (
     import_from_mixin, newlist_hint, resizelist_hint, specialize)
-from rpython.rlib.buffer import Buffer
 from rpython.rlib.rstring import StringBuilder, ByteListBuilder
-from rpython.rlib.debug import check_list_of_chars
+from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list,
                               ll_for_resizable_list)
-
+from rpython.rlib import jit
+from rpython.rlib.buffer import Buffer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
-from pypy.interpreter.signature import Signature
 from pypy.interpreter.typedef import TypeDef
-from pypy.objspace.std.sliceobject import W_SliceObject
+from pypy.interpreter.buffer import SimpleView
+from pypy.objspace.std.sliceobject import W_SliceObject, unwrap_start_stop
 from pypy.objspace.std.stringmethods import StringMethods, _get_buffer
+from pypy.objspace.std.stringmethods import _descr_getslice_slowpath

From pypy.commits at gmail.com  Mon May 15 08:27:11 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 05:27:11 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix tests
Message-ID: <59199e9f.eb97500a.4298f.4721@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91296:87a6a1665f4c
Date: 2017-05-15 14:03 +0200
http://bitbucket.org/pypy/pypy/changeset/87a6a1665f4c/

Log:	fix tests

diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -8,7 +8,7 @@
 
 class MyRawBuffer(RawBuffer):
 
-    def __init__(self, data, readonly):
+    def __init__(self, data, readonly=True):
         self.readonly = readonly
         self._n = len(data)
         self._buf = lltype.malloc(rffi.CCHARP.TO, self._n, flavor='raw')

From pypy.commits at gmail.com  Mon May 15 08:27:13 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 05:27:13 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix
 BytearrayBuffer.typed_{read, write}
Message-ID: <59199ea1.33ba500a.2b1ce.f3c2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91297:a1412ae7916c
Date: 2017-05-15 14:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a1412ae7916c/

Log:	fix BytearrayBuffer.typed_{read,write}

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1304,7 +1304,7 @@
 
     def _get_gc_data(self):
         from rpython.rtyper.lltypesystem import lltype, llmemory
-        ll_data = ll_for_resizable_list(self.data)
+        ll_data = ll_for_resizable_list(self.ba._data)
         ll_items = ll_data.items
         LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
         base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)

From pypy.commits at gmail.com  Mon May 15 09:19:51 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 06:19:51 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix for the recent changes to
 Bytearray,
 which has got an extra _offset field now. This code is a bit suboptimal
 because it contains lots of guards :(
Message-ID: <5919aaf7.efb4500a.2444e.ef0d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91298:f1c2d4165559
Date: 2017-05-15 15:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f1c2d4165559/

Log:	fix for the recent changes to Bytearray, which has got an extra
	_offset field now. This code is a bit suboptimal because it contains
	lots of guards :(

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -181,13 +181,15 @@
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('pack_into', """\
             guard_not_invalidated(descr=...)
-            p66 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst_data \d+>)
-            i67 = getfield_gc_i(p66, descr=<FieldS list.length \d+>)
-            i69 = int_sub(i67, 4)
-            i71 = int_lt(i69, 2)
-            guard_false(i71, descr=...)
-            i73 = int_le(i60, 32767)
-            guard_true(i73, descr=...)
-            p74 = getfield_gc_r(p66, descr=<FieldP list.items \d+>)
-            gc_store_indexed(p74, 4, i60, 1, _, 2, descr=<ArrayS 2>)
+            p68 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__data \d+>)
+            i69 = getfield_gc_i(p68, descr=<FieldS list.length \d+>)
+            i70 = getfield_gc_i(p14, descr=<FieldS pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__offset \d+>)
+            i71 = int_sub(i69, i70)
+            i73 = int_sub(i71, 4)
+            i75 = int_lt(i73, 2)
+            guard_false(i75, descr=...)
+            i77 = int_le(i62, 32767)
+            guard_true(i77, descr=...)
+            p78 = getfield_gc_r(p68, descr=<FieldP list.items \d+>)
+            gc_store_indexed(p78, 4, i62, 1, _, 2, descr=<ArrayS 2>)
         """)

From pypy.commits at gmail.com  Mon May 15 11:09:55 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 08:09:55 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: the Bytebuffer class in
 rlib.buffer and pypy.module.__pypy__ was almost identical. Kill the latter
Message-ID: <5919c4c3.8bae500a.8def6.cc2c@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91299:1c9b9bc8492e
Date: 2017-05-15 16:27 +0200
http://bitbucket.org/pypy/pypy/changeset/1c9b9bc8492e/

Log:	the Bytebuffer class in rlib.buffer and pypy.module.__pypy__ was
	almost identical. Kill the latter

diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,29 +2,8 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import ByteBuffer
 from pypy.interpreter.gateway import unwrap_spec
-from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
-
-
-class ByteBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, len):
-        self.data = ['\x00'] * len
-        self.readonly = False
-
-    def getlength(self):
-        return len(self.data)
-
-    def getitem(self, index):
-        return self.data[index]
-
-    def setitem(self, index, char):
-        self.data[index] = char
-
-    def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):

From pypy.commits at gmail.com  Mon May 15 11:09:59 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 08:09:59 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: introduce GCBuffer,
 to share the implementation of typed_read and type_write among all
 buffers which are baked by GC-managed memory. Use it for StringBuffer and
 BytearrayBuffer
Message-ID: <5919c4c7.44b1500a.e5985.9abd@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91301:d59317a1258f
Date: 2017-05-15 16:52 +0200
http://bitbucket.org/pypy/pypy/changeset/d59317a1258f/

Log:	introduce GCBuffer, to share the implementation of typed_read and
	type_write among all buffers which are baked by GC-managed memory.
	Use it for StringBuffer and BytearrayBuffer

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -9,7 +9,7 @@
                               nonmoving_raw_ptr_for_resizable_list,
                               ll_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import GCBuffer
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -1257,7 +1257,7 @@
         start += step
 
 
-class BytearrayBuffer(Buffer):
+class BytearrayBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, ba, readonly=False):
@@ -1308,25 +1308,7 @@
         ll_items = ll_data.items
         LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
         base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
-        scale_factor = llmemory.sizeof(lltype.Char)
-        return ll_items, scale_factor, base_ofs
-
-    @specialize.ll_and_arg(1)
-    def typed_read(self, TP, byte_offset):
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        ll_items, scale_factor, base_ofs = self._get_gc_data()
-        return llop.gc_load_indexed(TP, ll_items, byte_offset,
-                                    scale_factor, base_ofs)
-
-    @specialize.ll_and_arg(1)
-    def typed_write(self, TP, byte_offset, value):
-        from rpython.rtyper.lltypesystem import lltype
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        ll_items, scale_factor, base_ofs = self._get_gc_data()
-        value = lltype.cast_primitive(TP, value)
-        return llop.gc_store_indexed(lltype.Void, ll_items, byte_offset, value,
-                                     scale_factor, base_ofs)
-
+        return ll_items, base_ofs
 
 
 @specialize.argtype(1)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -26,7 +26,13 @@
     """
 
 class Buffer(object):
-    """Base class for buffers of bytes"""
+    """
+    Base class for buffers of bytes.
+
+    Most probably, you do NOT want to use this as a base class, but either
+    GCBuffer or RawBuffer, so that you automatically get the proper
+    implementation of typed_read and typed_write.
+    """
     _attrs_ = ['readonly']
     _immutable_ = True
 
@@ -89,7 +95,8 @@
 class RawBuffer(Buffer):
     """
     A buffer which is baked by a raw, non-movable memory area. It implementes
-    typed_read in terms of get_raw_address()
+    typed_read and typed_write in terms of get_raw_address(), llop.raw_load,
+    llop.raw_store.
 
     NOTE: this assumes that get_raw_address() is cheap. Do not use this as a
     base class if get_raw_address() is potentially costly, like for example if
@@ -117,6 +124,49 @@
         return llop.raw_store(lltype.Void, ptr, byte_offset, value)
 
 
+class GCBuffer(Buffer):
+    """
+    A buffer which is baked by a GC-managed memory area. It implements
+    typed_read and typed_write in terms of llop.gc_load_indexed and
+    llop.gc_store_indexed.
+
+    Subclasses MUST override the _get_gc_data method.
+    """
+    _immutable_ = True
+    _attrs_ = ['readonly']
+
+    def _get_gc_data(self):
+        """
+        Return a tuple (data, base_offset), whose items can be used with
+        llop.gc_{load,store}_indexed.
+        """
+        raise NotImplementedError
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        """
+        Read the value of type TP starting at byte_offset. No bounds checks
+        """
+        lldata, base_ofs = self._get_gc_data()
+        scale_factor = llmemory.sizeof(lltype.Char)
+        return llop.gc_load_indexed(TP, lldata, byte_offset,
+                                    scale_factor, base_ofs)
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        """
+        Write the value of type TP at byte_offset. No bounds checks
+        """
+        if self.readonly:
+            raise CannotWrite
+        lldata, base_ofs = self._get_gc_data()
+        scale_factor = llmemory.sizeof(lltype.Char)
+        value = lltype.cast_primitive(TP, value)
+        return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
+                                     scale_factor, base_ofs)
+
+
+
 class ByteBuffer(Buffer):
     _immutable_ = True
 
@@ -137,7 +187,7 @@
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 
-class StringBuffer(Buffer):
+class StringBuffer(GCBuffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
 
@@ -170,17 +220,11 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
-    @specialize.ll_and_arg(1)
-    def typed_read(self, TP, byte_offset):
-        # WARNING: the 'byte_offset' is, as its name says, measured in bytes;
-        # however, it should be aligned for TP, otherwise on some platforms this
-        # code will crash!
+    def _get_gc_data(self):
         lls = llstr(self.value)
         base_ofs = (llmemory.offsetof(STR, 'chars') +
                     llmemory.itemoffsetof(STR.chars, 0))
-        scale_factor = llmemory.sizeof(lltype.Char)
-        return llop.gc_load_indexed(TP, lls, byte_offset,
-                                    scale_factor, base_ofs)
+        return lls, base_ofs
 
 
 class SubBuffer(Buffer):

From pypy.commits at gmail.com  Mon May 15 11:10:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 08:10:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: write tests for ByteBuffer,
 make it a subclass of GCBuffer and reduce a bit of code duplication
 with ByteArrayBuffer
Message-ID: <5919c4c9.9da4500a.d7c2b.b298@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91302:261fbab29544
Date: 2017-05-15 17:08 +0200
http://bitbucket.org/pypy/pypy/changeset/261fbab29544/

Log:	write tests for ByteBuffer, make it a subclass of GCBuffer and
	reduce a bit of code duplication with ByteArrayBuffer

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -6,10 +6,9 @@
 from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-                              nonmoving_raw_ptr_for_resizable_list,
-                              ll_for_resizable_list)
+                              nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import GCBuffer
+from rpython.rlib.buffer import GCBuffer, get_gc_data_for_list_of_chars
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -1303,12 +1302,7 @@
         return p
 
     def _get_gc_data(self):
-        from rpython.rtyper.lltypesystem import lltype, llmemory
-        ll_data = ll_for_resizable_list(self.ba._data)
-        ll_items = ll_data.items
-        LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
-        base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
-        return ll_items, base_ofs
+        return get_gc_data_for_list_of_chars(self.ba._data)
 
 
 @specialize.argtype(1)
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -8,7 +8,8 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-                              nonmoving_raw_ptr_for_resizable_list)
+                              nonmoving_raw_ptr_for_resizable_list,
+                              ll_for_resizable_list)
 from rpython.rlib.signature import signature
 from rpython.rlib import types
 
@@ -166,8 +167,15 @@
                                      scale_factor, base_ofs)
 
 
+def get_gc_data_for_list_of_chars(data):
+    ll_data = ll_for_resizable_list(data)
+    ll_items = ll_data.items
+    LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
+    base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
+    return ll_items, base_ofs
 
-class ByteBuffer(Buffer):
+
+class ByteBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, n):
@@ -186,6 +194,9 @@
     def get_raw_address(self):
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
+    def _get_gc_data(self):
+        return get_gc_data_for_list_of_chars(self.data)
+
 
 class StringBuffer(GCBuffer):
     _attrs_ = ['readonly', 'value']
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -1,7 +1,8 @@
 import struct
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import r_singlefloat
-from rpython.rlib.buffer import StringBuffer, SubBuffer, Buffer, RawBuffer
+from rpython.rlib.buffer import (StringBuffer, SubBuffer, Buffer, RawBuffer,
+                                 ByteBuffer)
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 from rpython.rtyper.test.tool import BaseRtypingTest
@@ -101,7 +102,6 @@
         x = self.read(lltype.SingleFloat, buf, size)
         assert x == r_singlefloat(45.6)
 
-
 class TestTypedReadDirect(BaseTypedReadTest):
 
     def read(self, TYPE, data, offset):
@@ -157,3 +157,28 @@
         fn = self.cache[TYPE]
         x = fn(data, offset)
         return lltype.cast_primitive(TYPE, x)
+
+
+class TestByteBuffer(object):
+
+    def test_basic(self):
+        buf = ByteBuffer(4)
+        assert buf.getlength() == 4
+        assert buf.getitem(2) == '\x00'
+        buf.setitem(0, 'A')
+        buf.setitem(3, 'Z')
+        assert buf.as_str() == 'A\x00\x00Z'
+
+    def test_typed_write(self):
+        buf = ByteBuffer(4)
+        buf.typed_write(rffi.USHORT, 0, 0x1234)
+        buf.typed_write(rffi.USHORT, 2, 0x5678)
+        expected = struct.pack('HH', 0x1234, 0x5678)
+        assert buf.as_str() == expected
+    
+    def test_typed_read(self):
+        data = struct.pack('HH', 0x1234, 0x5678)
+        buf = ByteBuffer(4)
+        buf.setslice(0, data)
+        assert buf.typed_read(rffi.USHORT, 0) == 0x1234
+        assert buf.typed_read(rffi.USHORT, 2) == 0x5678

From pypy.commits at gmail.com  Mon May 15 11:09:57 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 08:09:57 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: move the definition of the
 class after RawBuffer,
 which is a base class and thus is better positioned near the top of the file
Message-ID: <5919c4c5.6cba500a.9b16c.e322@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91300:6a3872237846
Date: 2017-05-15 16:28 +0200
http://bitbucket.org/pypy/pypy/changeset/6a3872237846/

Log:	move the definition of the class after RawBuffer, which is a base
	class and thus is better positioned near the top of the file

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -86,27 +86,6 @@
         raise CannotWrite
 
 
-class ByteBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, n):
-        self.data = resizable_list_supporting_raw_ptr(['\0'] * n)
-        self.readonly = False
-
-    def getlength(self):
-        return len(self.data)
-
-    def getitem(self, index):
-        return self.data[index]
-
-    def setitem(self, index, char):
-        self.data[index] = char
-
-    def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
-
-
-
 class RawBuffer(Buffer):
     """
     A buffer which is baked by a raw, non-movable memory area. It implementes
@@ -138,6 +117,26 @@
         return llop.raw_store(lltype.Void, ptr, byte_offset, value)
 
 
+class ByteBuffer(Buffer):
+    _immutable_ = True
+
+    def __init__(self, n):
+        self.data = resizable_list_supporting_raw_ptr(['\0'] * n)
+        self.readonly = False
+
+    def getlength(self):
+        return len(self.data)
+
+    def getitem(self, index):
+        return self.data[index]
+
+    def setitem(self, index, char):
+        self.data[index] = char
+
+    def get_raw_address(self):
+        return nonmoving_raw_ptr_for_resizable_list(self.data)
+
+
 class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True

From pypy.commits at gmail.com  Mon May 15 11:13:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 15 May 2017 08:13:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: bah
Message-ID: <5919c57d.da8e500a.e3d6c.5082@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91303:bc40c2c21a98
Date: 2017-05-15 17:11 +0200
http://bitbucket.org/pypy/pypy/changeset/bc40c2c21a98/

Log:	bah

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1286,7 +1286,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return GCBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.

From pypy.commits at gmail.com  Tue May 16 05:40:29 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:40:29 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rpython fix,
 else the types returned by _get_gc_data conflict
Message-ID: <591ac90d.cf9adf0a.61b15.5b10@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91304:3e7adaec029c
Date: 2017-05-15 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/3e7adaec029c/

Log:	rpython fix, else the types returned by _get_gc_data conflict

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -172,6 +172,7 @@
     ll_items = ll_data.items
     LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
     base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
+    ll_items = lltype.cast_opaque_ptr(llmemory.GCREF, ll_items)
     return ll_items, base_ofs
 
 
@@ -233,6 +234,7 @@
 
     def _get_gc_data(self):
         lls = llstr(self.value)
+        lls = lltype.cast_opaque_ptr(llmemory.GCREF, lls)
         base_ofs = (llmemory.offsetof(STR, 'chars') +
                     llmemory.itemoffsetof(STR.chars, 0))
         return lls, base_ofs

From pypy.commits at gmail.com  Tue May 16 05:40:31 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:40:31 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: bah,
 the base_ofs needs to be proven constant at translation time. Try
 to help the rtyper
Message-ID: <591ac90f.95b81c0a.a5243.1cff@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91305:f23167f0f132
Date: 2017-05-15 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/f23167f0f132/

Log:	bah, the base_ofs needs to be proven constant at translation time.
	Try to help the rtyper

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -8,7 +8,8 @@
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import GCBuffer, get_gc_data_for_list_of_chars
+from rpython.rlib.buffer import (GCBuffer, get_gc_data_for_list_of_chars,
+                                 get_gc_data_offset_for_list_of_chars)
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -1304,6 +1305,9 @@
     def _get_gc_data(self):
         return get_gc_data_for_list_of_chars(self.ba._data)
 
+    def _get_gc_data_offset(self):
+        return get_gc_data_offset_for_list_of_chars()
+
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -4,6 +4,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rlist import LIST_OF
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib import jit
@@ -131,14 +132,17 @@
     typed_read and typed_write in terms of llop.gc_load_indexed and
     llop.gc_store_indexed.
 
-    Subclasses MUST override the _get_gc_data method.
+    Subclasses MUST override the _get_gc_* methods.
     """
     _immutable_ = True
     _attrs_ = ['readonly']
 
     def _get_gc_data(self):
+        raise NotImplementedError
+
+    def _get_gc_data_offset(self):
         """
-        Return a tuple (data, base_offset), whose items can be used with
+        Return the offset to use with _get_gc_data() for calling
         llop.gc_{load,store}_indexed.
         """
         raise NotImplementedError
@@ -148,7 +152,8 @@
         """
         Read the value of type TP starting at byte_offset. No bounds checks
         """
-        lldata, base_ofs = self._get_gc_data()
+        lldata = self._get_gc_data()
+        base_ofs = self._get_gc_data_offset()
         scale_factor = llmemory.sizeof(lltype.Char)
         return llop.gc_load_indexed(TP, lldata, byte_offset,
                                     scale_factor, base_ofs)
@@ -160,7 +165,8 @@
         """
         if self.readonly:
             raise CannotWrite
-        lldata, base_ofs = self._get_gc_data()
+        lldata = self._get_gc_data()
+        base_ofs = self._get_gc_data_offset()
         scale_factor = llmemory.sizeof(lltype.Char)
         value = lltype.cast_primitive(TP, value)
         return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
@@ -170,10 +176,11 @@
 def get_gc_data_for_list_of_chars(data):
     ll_data = ll_for_resizable_list(data)
     ll_items = ll_data.items
-    LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
-    base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
-    ll_items = lltype.cast_opaque_ptr(llmemory.GCREF, ll_items)
-    return ll_items, base_ofs
+    return lltype.cast_opaque_ptr(llmemory.GCREF, ll_items)
+
+def get_gc_data_offset_for_list_of_chars():
+    LIST = LIST_OF(lltype.Char)
+    return llmemory.itemoffsetof(LIST.items.TO, 0)
 
 
 class ByteBuffer(GCBuffer):
@@ -198,6 +205,9 @@
     def _get_gc_data(self):
         return get_gc_data_for_list_of_chars(self.data)
 
+    def _get_gc_data_offset(self):
+        return get_gc_data_offset_for_list_of_chars()
+
 
 class StringBuffer(GCBuffer):
     _attrs_ = ['readonly', 'value']
@@ -232,12 +242,13 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
+    def _get_gc_data_offset(self):
+        return (llmemory.offsetof(STR, 'chars') +
+                llmemory.itemoffsetof(STR.chars, 0))
+
     def _get_gc_data(self):
         lls = llstr(self.value)
-        lls = lltype.cast_opaque_ptr(llmemory.GCREF, lls)
-        base_ofs = (llmemory.offsetof(STR, 'chars') +
-                    llmemory.itemoffsetof(STR.chars, 0))
-        return lls, base_ofs
+        return lltype.cast_opaque_ptr(llmemory.GCREF, lls)
 
 
 class SubBuffer(Buffer):

From pypy.commits at gmail.com  Tue May 16 05:40:33 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:40:33 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rewrite buffer.GCBuffer as a
 class decorator instead of a base class,
 because we need to provide different, specialized versions of typed_{read,
 write} in which base_ofs is a constant to make the codewriter happy
Message-ID: <591ac911.110d1c0a.d14dd.7b59@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91306:67ba2f0639a7
Date: 2017-05-16 11:39 +0200
http://bitbucket.org/pypy/pypy/changeset/67ba2f0639a7/

Log:	rewrite buffer.GCBuffer as a class decorator instead of a base
	class, because we need to provide different, specialized versions of
	typed_{read,write} in which base_ofs is a constant to make the
	codewriter happy

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -8,7 +8,8 @@
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import (GCBuffer, get_gc_data_for_list_of_chars,
+from rpython.rlib.buffer import (Buffer, GCBuffer,
+                                 get_gc_data_for_list_of_chars,
                                  get_gc_data_offset_for_list_of_chars)
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -1257,7 +1258,8 @@
         start += step
 
 
-class BytearrayBuffer(GCBuffer):
+ at GCBuffer
+class BytearrayBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, ba, readonly=False):
@@ -1302,12 +1304,13 @@
         p = rffi.ptradd(p, ba._offset)
         return p
 
+    @staticmethod
+    def _get_gc_data_offset():
+        return get_gc_data_offset_for_list_of_chars()
+
     def _get_gc_data(self):
         return get_gc_data_for_list_of_chars(self.ba._data)
 
-    def _get_gc_data_offset(self):
-        return get_gc_data_offset_for_list_of_chars()
-
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -31,9 +31,10 @@
     """
     Base class for buffers of bytes.
 
-    Most probably, you do NOT want to use this as a base class, but either
-    GCBuffer or RawBuffer, so that you automatically get the proper
-    implementation of typed_read and typed_write.
+    Most probably, you do NOT want to use this as a lone base class, but
+    either inherit from RawBuffer or use the GCBuffer class decorator, so that
+    you automatically get the proper implementation of typed_read and
+    typed_write.
     """
     _attrs_ = ['readonly']
     _immutable_ = True
@@ -126,52 +127,46 @@
         return llop.raw_store(lltype.Void, ptr, byte_offset, value)
 
 
-class GCBuffer(Buffer):
+def GCBuffer(buffercls):
     """
-    A buffer which is baked by a GC-managed memory area. It implements
-    typed_read and typed_write in terms of llop.gc_load_indexed and
-    llop.gc_store_indexed.
+    class decorator for a buffer which is baked by a GC-managed memory
+    area. It implements typed_read and typed_write in terms of
+    llop.gc_load_indexed and llop.gc_store_indexed.
 
-    Subclasses MUST override the _get_gc_* methods.
-    """
-    _immutable_ = True
-    _attrs_ = ['readonly']
+    The target class MUST provide the following methods:
+
+    @staticmethod
+    def _get_gc_data_offset(self):
+        raise NotImplementedError
 
     def _get_gc_data(self):
         raise NotImplementedError
-
-    def _get_gc_data_offset(self):
-        """
-        Return the offset to use with _get_gc_data() for calling
-        llop.gc_{load,store}_indexed.
-        """
-        raise NotImplementedError
+    """
+    # We had to write this as a class decorator instead of a base class
+    # because we need to produce specialized versions of typed_{read,write} in
+    # which base_ofs is an RPython constant.
+    base_ofs = buffercls._get_gc_data_offset()
+    scale_factor = llmemory.sizeof(lltype.Char)
 
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
-        """
-        Read the value of type TP starting at byte_offset. No bounds checks
-        """
         lldata = self._get_gc_data()
-        base_ofs = self._get_gc_data_offset()
-        scale_factor = llmemory.sizeof(lltype.Char)
         return llop.gc_load_indexed(TP, lldata, byte_offset,
                                     scale_factor, base_ofs)
 
     @specialize.ll_and_arg(1)
     def typed_write(self, TP, byte_offset, value):
-        """
-        Write the value of type TP at byte_offset. No bounds checks
-        """
         if self.readonly:
             raise CannotWrite
         lldata = self._get_gc_data()
-        base_ofs = self._get_gc_data_offset()
-        scale_factor = llmemory.sizeof(lltype.Char)
         value = lltype.cast_primitive(TP, value)
         return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
                                      scale_factor, base_ofs)
 
+    buffercls.typed_read = typed_read
+    buffercls.typed_write = typed_write
+    return buffercls
+
 
 def get_gc_data_for_list_of_chars(data):
     ll_data = ll_for_resizable_list(data)
@@ -183,7 +178,8 @@
     return llmemory.itemoffsetof(LIST.items.TO, 0)
 
 
-class ByteBuffer(GCBuffer):
+ at GCBuffer
+class ByteBuffer(Buffer):
     _immutable_ = True
 
     def __init__(self, n):
@@ -205,11 +201,13 @@
     def _get_gc_data(self):
         return get_gc_data_for_list_of_chars(self.data)
 
-    def _get_gc_data_offset(self):
+    @staticmethod
+    def _get_gc_data_offset():
         return get_gc_data_offset_for_list_of_chars()
 
 
-class StringBuffer(GCBuffer):
+ at GCBuffer
+class StringBuffer(Buffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
 
@@ -242,7 +240,8 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
-    def _get_gc_data_offset(self):
+    @staticmethod
+    def _get_gc_data_offset():
         return (llmemory.offsetof(STR, 'chars') +
                 llmemory.itemoffsetof(STR.chars, 0))
 
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -6,6 +6,8 @@
 from rpython.annotator.annrpython import RPythonAnnotator
 from rpython.annotator.model import SomeInteger
 from rpython.rtyper.test.tool import BaseRtypingTest
+from rpython.jit.metainterp.test.support import LLJitMixin
+
 
 class MyRawBuffer(RawBuffer):
 
@@ -182,3 +184,33 @@
         buf.setslice(0, data)
         assert buf.typed_read(rffi.USHORT, 0) == 0x1234
         assert buf.typed_read(rffi.USHORT, 2) == 0x5678
+
+
+class TestJIT(LLJitMixin):
+
+    def test_GCBuffer_typed_read(self):
+        from rpython.rlib import jit
+        DATA = struct.pack('i', 0x12345678)
+
+        @jit.dont_look_inside
+        def make_buffer(flag):
+            if flag:
+                buf = ByteBuffer(len(DATA))
+                buf.setslice(0, DATA)
+            else:
+                buf = StringBuffer(DATA)
+            return buf
+
+        def f(flag):
+            buf = make_buffer(flag)
+            return buf.typed_read(rffi.INT, 0)
+
+        for flag in (0, 1):
+            res = self.interp_operations(f, [0], supports_singlefloats=True)
+            #
+            self.check_operations_history({'call_r': 1,
+                                           'guard_no_exception': 1,
+                                           'guard_class': 1,
+                                           'gc_load_indexed_i': 1,
+                                           'finish': 1})
+            assert res == 0x12345678

From pypy.commits at gmail.com  Tue May 16 05:49:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:49:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix tests
Message-ID: <591acb0d.028b1c0a.af12f.b296@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91307:58756fc27659
Date: 2017-05-16 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/58756fc27659/

Log:	fix tests

diff --git a/rpython/rlib/test/test_mutbuffer.py b/rpython/rlib/test/test_mutbuffer.py
--- a/rpython/rlib/test/test_mutbuffer.py
+++ b/rpython/rlib/test/test_mutbuffer.py
@@ -7,6 +7,7 @@
 
     def test_finish(self):
         buf = MutableStringBuffer(4)
+        buf.setzeros(0, 4)
         pytest.raises(ValueError, "buf.as_str()")
         s = buf.finish()
         assert s == '\x00' * 4
@@ -22,6 +23,7 @@
 
     def test_setslice(self):
         buf = MutableStringBuffer(6)
+        buf.setzeros(0, 6)
         buf.setslice(2, 'ABCD')
         assert buf.finish() == '\x00\x00ABCD'
 

From pypy.commits at gmail.com  Tue May 16 05:49:03 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:49:03 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: use @GCBuffer to implement
 MutableStringBuffer
Message-ID: <591acb0f.44881c0a.414d5.80a1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91308:967e8af6bfb8
Date: 2017-05-16 11:47 +0200
http://bitbucket.org/pypy/pypy/changeset/967e8af6bfb8/

Log:	use @GCBuffer to implement MutableStringBuffer

diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -3,9 +3,10 @@
 from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
 from rpython.rtyper.annlowlevel import llstr, hlstr
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import Buffer, GCBuffer
 from rpython.rlib import jit
 
+ at GCBuffer
 class MutableStringBuffer(Buffer):
     """
     A writeable buffer to incrementally fill a string of a fixed size.
@@ -53,11 +54,10 @@
         for i in range(index, index+count):
             self.setitem(i, '\x00')
 
-    @specialize.ll_and_arg(1)
-    def typed_write(self, TP, byte_offset, value):
-        base_ofs = (llmemory.offsetof(STR, 'chars') +
-                    llmemory.itemoffsetof(STR.chars, 0))
-        scale_factor = llmemory.sizeof(lltype.Char)
-        value = lltype.cast_primitive(TP, value)
-        llop.gc_store_indexed(lltype.Void, self.ll_val, byte_offset, value,
-                              scale_factor, base_ofs)
+    @staticmethod
+    def _get_gc_data_offset():
+        return (llmemory.offsetof(STR, 'chars') +
+                llmemory.itemoffsetof(STR.chars, 0))
+
+    def _get_gc_data(self):
+        return lltype.cast_opaque_ptr(llmemory.GCREF, self.ll_val)

From pypy.commits at gmail.com  Tue May 16 05:51:58 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 16 May 2017 02:51:58 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: bah
Message-ID: <591acbbe.c4b51c0a.ccadb.282c@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91309:eace635b2834
Date: 2017-05-16 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/eace635b2834/

Log:	bah

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1289,7 +1289,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return GCBuffer.getslice(self, start, stop, step, size)
+        return Buffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.

From pypy.commits at gmail.com  Tue May 16 06:54:34 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 16 May 2017 03:54:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test
Message-ID: <591ada6a.028b1c0a.af12f.ca47@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91312:e45afbe0ded7
Date: 2017-05-16 13:53 +0300
http://bitbucket.org/pypy/pypy/changeset/e45afbe0ded7/

Log:	fix test

diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -298,7 +298,7 @@
                         while (PyDict_Next(arg, &pos, &key, &value))
                             ret ++;
                         /* test no crash if pos is not reset to 0*/
-                        while (PyDict_Next(args, &pos, &key, &value))
+                        while (PyDict_Next(arg, &pos, &key, &value))
                             ret ++;
                     }
                 }

From pypy.commits at gmail.com  Tue May 16 06:54:30 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 16 May 2017 03:54:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: add test that passes with -A
Message-ID: <591ada66.d296df0a.0413.b71e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91310:ad45faa583f4
Date: 2017-05-15 22:42 +0300
http://bitbucket.org/pypy/pypy/changeset/ad45faa583f4/

Log:	add test that passes with -A

diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -255,4 +255,60 @@
             ])
         d = module.get_type_dict(1)
         assert d['real'].__get__(1, 1) == 1
-
+    def test_advanced(self):
+        module = self.import_extension('foo', [
+            ("dict_len", "METH_O",
+            '''
+                int ret = args->ob_type->tp_as_mapping->mp_length(args);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_setitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 3 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1),
+                        PyTuple_GetItem(args, 2));
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_delitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 2 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1), NULL);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_next", "METH_VARARGS",
+            '''
+                PyObject *key, *value;
+                PyObject *arg = NULL;
+                Py_ssize_t pos = 0;
+                int ret = 0;
+                if ((PyArg_ParseTuple(args, "|O", &arg))) {
+                    if (arg && PyDict_Check(arg)) {
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                        /* test no crash if pos is not reset to 0*/
+                        while (PyDict_Next(args, &pos, &key, &value))
+                            ret ++;
+                    }
+                }
+                return PyLong_FromLong(ret);
+            '''),
+            ])
+        d = {'a': 1, 'b':2}
+        assert module.dict_len(d) == 2
+        assert module.dict_setitem(d, 'a', 'c') == 0
+        assert d['a'] == 'c'
+        assert module.dict_delitem(d, 'a') == 0
+        r = module.dict_next({'a': 1, 'b': 2})
+        assert r == 2

From pypy.commits at gmail.com  Tue May 16 06:54:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 16 May 2017 03:54:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: add slot definitions for __len__,
 __setitem__, __delitem__, avoid crash in PyDict_Next
Message-ID: <591ada68.4699df0a.10ec3.ab3f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91311:fbcdfc92a521
Date: 2017-05-16 13:52 +0300
http://bitbucket.org/pypy/pypy/changeset/fbcdfc92a521/

Log:	add slot definitions for __len__, __setitem__, __delitem__, avoid
	crash in PyDict_Next

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
 
     if w_dict is None:
         return 0
-
+    if not space.isinstance_w(w_dict, space.w_dict):
+        return 0 
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         Py_IncRef(space, py_dict.c__tmpkeys)
     else:
+        if not py_dict.c__tmpkeys:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
         w_keys = from_ref(space, py_dict.c__tmpkeys)
     ppos[0] += 1
     if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,7 +14,7 @@
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
@@ -265,7 +265,7 @@
     check_num_args(space, w_args, 1)
     args_w = space.fixedview(w_args)
     index = space.int_w(space.index(args_w[0]))
-    null = lltype.nullptr(PyObject.TO)
+    null = rffi.cast(PyObject, 0)
     res = generic_cpy_call(space, func_target, w_self, index, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
@@ -294,7 +294,8 @@
     func_target = rffi.cast(objobjargproc, func)
     check_num_args(space, w_args, 1)
     w_key, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, None)
+    null = rffi.cast(PyObject, 0)
+    res = generic_cpy_call(space, func_target, w_self, w_key, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.w_None
@@ -612,6 +613,8 @@
             handled = True
 
     for tp_name, attr in [('tp_hash', '__hash__'),
+                          ('tp_as_sequence.c_sq_length', '__len__'),
+                          ('tp_as_mapping.c_mp_length', '__len__'),
                          ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -637,7 +640,8 @@
                           ('tp_as_number.c_nb_xor', '__xor__'),
                           ('tp_as_number.c_nb_or', '__or__'),
                           ('tp_as_sequence.c_sq_concat', '__add__'),
-                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+                          ('tp_as_mapping.c_mp_subscript', '__getitem__'),
                           ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -651,7 +655,7 @@
             handled = True
 
     # binary-with-Py_ssize_t-type
-    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -680,7 +684,48 @@
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
             handled = True
+    # ternary-with-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
 
+            @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, w_arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, w_arg1)
+                return 0
+            handled = True
+    # ternary-Py_size_t-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
+
+            @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, space.newint(arg1))
+                return 0
+            handled = True
     if handled:
         pass
     elif name == 'tp_setattro':

From pypy.commits at gmail.com  Tue May 16 10:18:28 2017
From: pypy.commits at gmail.com (danchr)
Date: Tue, 16 May 2017 07:18:28 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use correct keyword argument for
 strftime() routines in datetime
Message-ID: <591b0a34.4e901c0a.edc4e.f615@mx.google.com>

Author: Dan Villiom Podlaski Christiansen <dan at magenta.dk>
Branch: py3.5
Changeset: r91313:4e110113b03d
Date: 2017-05-16 16:12 +0200
http://bitbucket.org/pypy/pypy/changeset/4e110113b03d/

Log:	Use correct keyword argument for strftime() routines in datetime

	The argument is specified as 'format' on docs.python.org, and this
	is the argument pypy2 uses. For some reason or other, pypy3 always
	used 'fmt'.

diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -746,15 +746,15 @@
             _MONTHNAMES[self._month],
             self._day, self._year)
 
-    def strftime(self, fmt):
+    def strftime(self, format):
         "Format using strftime()."
-        return _wrap_strftime(self, fmt, self.timetuple())
+        return _wrap_strftime(self, format, self.timetuple())
 
-    def __format__(self, fmt):
-        if not isinstance(fmt, str):
-            raise TypeError("must be str, not %s" % type(fmt).__name__)
-        if len(fmt) != 0:
-            return self.strftime(fmt)
+    def __format__(self, format):
+        if not isinstance(format, str):
+            raise TypeError("must be str, not %s" % type(format).__name__)
+        if len(format) != 0:
+            return self.strftime(format)
         return str(self)
 
     def isoformat(self):
@@ -1215,7 +1215,7 @@
 
     __str__ = isoformat
 
-    def strftime(self, fmt):
+    def strftime(self, format):
         """Format using strftime().  The date part of the timestamp passed
         to underlying strftime should not be used.
         """
@@ -1224,13 +1224,13 @@
         timetuple = (1900, 1, 1,
                      self._hour, self._minute, self._second,
                      0, 1, -1)
-        return _wrap_strftime(self, fmt, timetuple)
+        return _wrap_strftime(self, format, timetuple)
 
-    def __format__(self, fmt):
-        if not isinstance(fmt, str):
-            raise TypeError("must be str, not %s" % type(fmt).__name__)
-        if len(fmt) != 0:
-            return self.strftime(fmt)
+    def __format__(self, format):
+        if not isinstance(format, str):
+            raise TypeError("must be str, not %s" % type(format).__name__)
+        if len(format) != 0:
+            return self.strftime(format)
         return str(self)
 
     # Timezone functions
diff --git a/lib-python/3/test/datetimetester.py b/lib-python/3/test/datetimetester.py
--- a/lib-python/3/test/datetimetester.py
+++ b/lib-python/3/test/datetimetester.py
@@ -1238,6 +1238,9 @@
         #check that this standard extension works
         t.strftime("%f")
 
+        #test that passing keyword arguments work
+        self.assertEqual(t.strftime(format=""), "")
+
     def test_format(self):
         dt = self.theclass(2007, 9, 10)
         self.assertEqual(dt.__format__(''), str(dt))
@@ -1602,6 +1605,9 @@
             self.assertEqual(a.__format__(fmt), dt.strftime(fmt))
             self.assertEqual(b.__format__(fmt), 'B')
 
+        #test that passing keyword arguments work
+        self.assertEqual(dt.strftime(format=""), "")
+
     def test_more_ctime(self):
         # Test fields that TestDate doesn't touch.
         import time
@@ -2335,6 +2341,9 @@
         # A naive object replaces %z and %Z with empty strings.
         self.assertEqual(t.strftime("'%z' '%Z'"), "'' ''")
 
+        #test that passing keyword arguments work
+        self.assertEqual(t.strftime(format=""), "")
+
     def test_format(self):
         t = self.theclass(1, 2, 3, 4)
         self.assertEqual(t.__format__(''), str(t))

From pypy.commits at gmail.com  Tue May 16 13:17:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 16 May 2017 10:17:29 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-recursionlimit: add test,
 implement recursion limit functions with ugly hack for untranslated
 test
Message-ID: <591b3429.02e81c0a.9bbca.1d36@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-recursionlimit
Changeset: r91314:9fa4ac53f85d
Date: 2017-05-16 20:16 +0300
http://bitbucket.org/pypy/pypy/changeset/9fa4ac53f85d/

Log:	add test, implement recursion limit functions with ugly hack for
	untranslated test

diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +228,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, limit)
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                    "maximum recursion depth exceeded" + rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                     "maximum recursion depth exceeded" + rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -490,29 +490,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyFileObject], lltype.Void)
 def PyFile_IncUseCount(space, p):
     """Increments the PyFileObject's internal use count to indicate
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -331,3 +331,30 @@
                 def nested_flags():
                     return module.get_flags()""" in ns
         assert ns['nested_flags']() == (1, 0x2000)  # CO_FUTURE_DIVISION
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  

From pypy.commits at gmail.com  Tue May 16 13:52:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 16 May 2017 10:52:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Call sys.__interactivehook__ (issue #2558)
Message-ID: <591b3c5e.41a0df0a.70612.5f93@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91315:94add67df5b3
Date: 2017-05-16 18:51 +0100
http://bitbucket.org/pypy/pypy/changeset/94add67df5b3/

Log:	Call sys.__interactivehook__ (issue #2558)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -694,6 +694,8 @@
                             del mainmodule.__file__
                         except (AttributeError, TypeError):
                             pass
+                if hasattr(sys, '__interactivehook__'):
+                    run_toplevel(sys.__interactivehook__)
                 # Then we need a prompt.
                 inspect = True
             else:

From pypy.commits at gmail.com  Tue May 16 15:27:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 16 May 2017 12:27:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Document PyBuffer
Message-ID: <591b52b9.cc9bdf0a.84d65.c521@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91316:0ab207b3359d
Date: 2017-05-16 20:27 +0100
http://bitbucket.org/pypy/pypy/changeset/0ab207b3359d/

Log:	Document PyBuffer

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -21,3 +21,8 @@
 .. branch: py3.5-mac-translate
 
 Fix for different posix primitives on MacOS
+
+.. branch: PyBuffer
+
+Internal refactoring of memoryviews and buffers, fixing some related
+performance issues.

From pypy.commits at gmail.com  Tue May 16 15:32:59 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 16 May 2017 12:32:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <591b53eb.53341c0a.3d8a0.6e6e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91317:a0e3281f3874
Date: 2017-05-16 20:32 +0100
http://bitbucket.org/pypy/pypy/changeset/a0e3281f3874/

Log:	fix test

diff --git a/pypy/interpreter/test/test_error.py b/pypy/interpreter/test/test_error.py
--- a/pypy/interpreter/test/test_error.py
+++ b/pypy/interpreter/test/test_error.py
@@ -118,10 +118,11 @@
     class FakeSpace:
         w_OSError = [OSError]
         w_EnvironmentError = [EnvironmentError]
+        w_None = None
         def wrap(self, obj):
             return [obj]
         newint = newtext = newunicode = newfilename = wrap
-        def call_function(self, exc, w_errno, w_msg, w_filename=None):
+        def call_function(self, exc, w_errno, w_msg, w_filename=None, *args):
             return (exc, w_errno, w_msg, w_filename)
     space = FakeSpace()
     #

From pypy.commits at gmail.com  Tue May 16 15:58:27 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 16 May 2017 12:58:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <591b59e3.229adf0a.cc8b.455b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91318:f74c6e0f3dd2
Date: 2017-05-16 20:57 +0100
http://bitbucket.org/pypy/pypy/changeset/f74c6e0f3dd2/

Log:	fix test

diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -217,7 +217,7 @@
                 gateway.interp2app(_cur_memory_pressure))
         else:
             def _skip_test(*ignored):
-                pytest.skip("not for -A testing")
+                skip("not for -A testing")
             cls.w_reset_memory_pressure = _skip_test
 
     def teardown_class(cls):

From pypy.commits at gmail.com  Tue May 16 16:08:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 16 May 2017 13:08:35 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-recursionlimit: fix translation
Message-ID: <591b5c43.902f1c0a.15036.1bbc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-recursionlimit
Changeset: r91319:7b9dd48c2572
Date: 2017-05-16 23:00 +0300
http://bitbucket.org/pypy/pypy/changeset/7b9dd48c2572/

Log:	fix translation

diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -2,6 +2,7 @@
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -236,7 +237,7 @@
 @cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
 def Py_SetRecursionLimit(space, limit):
     from pypy.module.sys.vm import setrecursionlimit
-    setrecursionlimit(space, limit)
+    setrecursionlimit(space, widen(limit))
 
 limit = 0 # for testing
 
@@ -261,12 +262,12 @@
         limit += 1
         if limit > 10:
             raise oefmt(space.w_RuntimeError, 
-                    "maximum recursion depth exceeded" + rffi.charp2str(where))
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
         return 0
     from rpython.rlib.rstack import stack_almost_full
     if stack_almost_full():
         raise oefmt(space.w_RuntimeError,
-                     "maximum recursion depth exceeded" + rffi.charp2str(where))
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
     return 0
 
 @cpython_api([], lltype.Void)

From pypy.commits at gmail.com  Wed May 17 11:31:48 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 17 May 2017 08:31:48 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Remove
 former transaction length back off mechanism and comment out debug output
Message-ID: <591c6ce4.6c8edf0a.dd814.37e4@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2061:5f7220351ad9
Date: 2017-05-12 17:56 +0200
http://bitbucket.org/pypy/stmgc/changeset/5f7220351ad9/

Log:	Remove former transaction length back off mechanism and comment out
	debug output

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1167,8 +1167,7 @@
     if (number_of_segments_in_use() < 2) {
         stm_become_inevitable(tl, "single thread mode");
     }
-    /* TODO remove: else, 'nursery_mark' was already set
-        in abort_data_structures_from_segment_num() */
+
     STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
                                         stm_get_transaction_length(tl));
     return repeat_count;
@@ -1532,22 +1531,6 @@
     if (pseg->active_queues)
         queues_deactivate_all(pseg, /*at_commit=*/false);
 
-
-    /* Set the next nursery_mark: first compute the value that
-       nursery_mark must have had at the start of the aborted transaction */
-    stm_char *old_mark =pseg->pub.nursery_mark + pseg->total_throw_away_nursery;
-
-    /* This means that the limit, in term of bytes, was: */
-    uintptr_t old_limit = old_mark - (stm_char *)_stm_nursery_start;
-
-    /* If 'total_throw_away_nursery' is smaller than old_limit, use that */
-    if (pseg->total_throw_away_nursery < old_limit)
-        old_limit = pseg->total_throw_away_nursery;
-
-    /* Now set the new limit to 90% of the old limit */
-    pseg->pub.nursery_mark = ((stm_char *)_stm_nursery_start +
-                              (uintptr_t)(old_limit * 0.9));
-
 #ifdef STM_NO_AUTOMATIC_SETJMP
     did_abort = 1;
 #endif
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -73,7 +73,7 @@
     }
     uintptr_t result = DEFAULT_FILL_MARK_NURSERY_BYTES +
         (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
-    printf("%020" PRIxPTR "\n", result);
+    // printf("%020" PRIxPTR "\n", result);
     return result;
 }
 

From pypy.commits at gmail.com  Wed May 17 11:31:50 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 17 May 2017 08:31:50 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Remove min
 transaction length (is now zero)
Message-ID: <591c6ce6.e9addf0a.173f4.6519@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2062:aa3ac5b4c94e
Date: 2017-05-16 16:57 +0200
http://bitbucket.org/pypy/stmgc/changeset/aa3ac5b4c94e/

Log:	Remove min transaction length (is now zero)

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -16,14 +16,11 @@
 static uintptr_t _stm_nursery_start;
 
 #define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4)
-// just double the size at max
+
 // #define LARGE_FILL_MARK_NURSERY_BYTES   DEFAULT_FILL_MARK_NURSERY_BYTES
 #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000L
 // #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
 
-uintptr_t stm_fill_mark_nursery_bytes = DEFAULT_FILL_MARK_NURSERY_BYTES;
-// uintptr_t stm_fill_mark_nursery_bytes = LARGE_FILL_MARK_NURSERY_BYTES;
-
 #define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001)
 
 static double get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
@@ -71,7 +68,7 @@
             STM_SINGLE_THREAD_MODE_ADAPTIVE,
             &stm_duration_payload);
     }
-    uintptr_t result = DEFAULT_FILL_MARK_NURSERY_BYTES +
+    uintptr_t result =
         (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
     // printf("%020" PRIxPTR "\n", result);
     return result;

From pypy.commits at gmail.com  Wed May 17 11:31:52 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 17 May 2017 08:31:52 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Initialize
 single thread mode properties of thread local in setup.c
Message-ID: <591c6ce8.d28bdf0a.7f3c2.bb20@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2063:6074cbac03a5
Date: 2017-05-17 12:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/6074cbac03a5/

Log:	Initialize single thread mode properties of thread local in setup.c

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -47,12 +47,7 @@
 }
 
 static void stm_transaction_length_handle_validation(stm_thread_local_t *tl, bool aborts) {
-    if (!tl->initialized) { // TODO move to setup.c
-        tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
-        tl->initialized = true;
-    }
-    double new = get_new_transaction_length(tl, aborts);
-    tl->relative_transaction_length = new;
+    tl->relative_transaction_length = get_new_transaction_length(tl, aborts);
 }
 
 static uintptr_t stm_get_transaction_length(stm_thread_local_t *tl) {
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -245,6 +245,11 @@
        numbers automatically. */
     tl->last_associated_segment_num = num + 1;
     tl->thread_local_counter = ++thread_local_counters;
+
+    /* init single thread mode */
+    tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
+    tl->transaction_length_backoff = 0;
+
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
     set_gs_register(get_segment_base(num + 1));
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -89,10 +89,11 @@
     struct stm_thread_local_s *prev, *next;
     intptr_t self_or_0_if_atomic;
     void *creating_pthread[2];
-    /* adaptive single thread mode */
+    /* == adaptive single thread mode == */
+    /* factor that is multiplied with max transaction length before the start of the next transaction on this thread */
     double relative_transaction_length;
+    /* when zero, transaction length may increase or decrease, otherwise transaction length may only decrease. is (re-)set to some value upon abort and counted down until zero upon successful validation. */
     int transaction_length_backoff;
-    bool initialized;
 } stm_thread_local_t;
 
 

From pypy.commits at gmail.com  Wed May 17 11:31:54 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 17 May 2017 08:31:54 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Do not set
 transactions inevitable on start
Message-ID: <591c6cea.61a9df0a.b1868.e3c9@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2064:9ca9e3e987a3
Date: 2017-05-17 13:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/9ca9e3e987a3/

Log:	Do not set transactions inevitable on start

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1164,10 +1164,6 @@
     }
     _do_start_transaction(tl);
 
-    if (number_of_segments_in_use() < 2) {
-        stm_become_inevitable(tl, "single thread mode");
-    }
-
     STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
                                         stm_get_transaction_length(tl));
     return repeat_count;

From pypy.commits at gmail.com  Wed May 17 11:31:56 2017
From: pypy.commits at gmail.com (tobweber)
Date: Wed, 17 May 2017 08:31:56 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Add timing
 macros to hide internals from user code when generating custom payload
 events
Message-ID: <591c6cec.85a2df0a.b47c.b8f6@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2065:b74e3a67f424
Date: 2017-05-17 14:56 +0200
http://bitbucket.org/pypy/stmgc/changeset/b74e3a67f424/

Log:	Add timing macros to hide internals from user code when generating
	custom payload events

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -52,17 +52,8 @@
 
 static uintptr_t stm_get_transaction_length(stm_thread_local_t *tl) {
     double relative_additional_length = tl->relative_transaction_length;
-    if (timing_enabled()) {
-        struct timespec relative_length = {
-            .tv_sec = (int)relative_additional_length,
-            .tv_nsec = (int)(fmod(relative_additional_length, 1) * 1000000000),
-        };
-        stm_duration_payload(relative_length);
-        stmcb_timing_event(
-            STM_SEGMENT->running_thread,
-            STM_SINGLE_THREAD_MODE_ADAPTIVE,
-            &stm_duration_payload);
-    }
+    publish_custom_value_event(
+        relative_additional_length, STM_SINGLE_THREAD_MODE_ADAPTIVE);
     uintptr_t result =
         (uintptr_t)(LARGE_FILL_MARK_NURSERY_BYTES * relative_additional_length);
     // printf("%020" PRIxPTR "\n", result);
diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -29,20 +29,31 @@
 
 #define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &duration_data };                                     \
+        { .duration = &(duration_data) };                                     \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 
 #define publish_event(thread_local, event)                                  \
     (timing_enabled() ?                                                     \
-        stmcb_timing_event(thread_local, event, &stm_duration_payload) :    \
+        stmcb_timing_event((thread_local), (event), &stm_duration_payload) :\
         (void)0);
 
 #define stop_timer_and_publish_for_thread(thread_local, event)              \
     pause_timer()                                                           \
     stm_duration_payload(duration)                                          \
-    assert(thread_local != NULL);                                           \
-    publish_event(thread_local, event)
+    assert((thread_local) != NULL);                                         \
+    publish_event((thread_local), (event))
 
 #define stop_timer_and_publish(event)                                       \
-    stop_timer_and_publish_for_thread(STM_SEGMENT->running_thread, event)
+    stop_timer_and_publish_for_thread(STM_SEGMENT->running_thread, (event))
+
+#define set_payload(double_value)                                           \
+    struct timespec payload_value = {                                       \
+        .tv_sec = (int)(double_value),                                      \
+        .tv_nsec = (int)(fmod((double_value), 1) * 1000000000),             \
+    };
+
+#define publish_custom_value_event(double_value, event)                     \
+    set_payload((double_value))                                             \
+    stm_duration_payload(payload_value);                                    \
+    publish_event(STM_SEGMENT->running_thread, (event))

From pypy.commits at gmail.com  Wed May 17 11:47:54 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 17 May 2017 08:47:54 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: move setzeros into the base
 class rlib.buffer.Buffer,
 because it is used by rstruct to pack and needs to be available on all
 buffers
Message-ID: <591c70aa.020a1c0a.466d0.e84b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91320:cce350f856ce
Date: 2017-05-17 11:48 +0200
http://bitbucket.org/pypy/pypy/changeset/cce350f856ce/

Log:	move setzeros into the base class rlib.buffer.Buffer, because it is
	used by rstruct to pack and needs to be available on all buffers

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -535,4 +535,9 @@
         buf = bytearray(len(expected))
         self.struct.pack_into("ii", buf, 0, 42, 43)
         assert buf == expected
-        
+
+    def test_pack_into_bytearray_padding(self):
+        expected = self.struct.pack("xxi", 42)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("xxi", buf, 0, 42)
+        assert buf == expected
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -80,6 +80,12 @@
         for i in range(len(string)):
             self.setitem(start + i, string[i])
 
+    @jit.look_inside_iff(lambda self, index, count:
+                         jit.isconstant(count) and count <= 8)
+    def setzeros(self, index, count):
+        for i in range(index, index+count):
+            self.setitem(i, '\x00')
+
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
         """
diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -48,12 +48,6 @@
     def setitem(self, index, char):
         self.ll_val.chars[index] = char
 
-    @jit.look_inside_iff(lambda self, index, count:
-                         jit.isconstant(count) and count <= 8)
-    def setzeros(self, index, count):
-        for i in range(index, index+count):
-            self.setitem(i, '\x00')
-
     @staticmethod
     def _get_gc_data_offset():
         return (llmemory.offsetof(STR, 'chars') +
diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -24,6 +24,10 @@
     def as_str(self):
         return rffi.charpsize2str(self._buf, self._n)
 
+    def setitem(self, i, char):
+        assert not self.readonly
+        self._buf[i] = char
+
     def __del__(self):
         lltype.free(self._buf, flavor='raw')
         self._buf = None
@@ -73,6 +77,13 @@
     assert addr[4] == b'o'
     assert addr[6] == b'w'
 
+def test_setzeros():
+    buf = MyRawBuffer('ABCDEFGH', readonly=False)
+    buf.setzeros(2, 3)
+    assert buf.as_str() == 'AB\x00\x00\x00FGH'
+
+    
+
 
 class BaseTypedReadTest:
 

From pypy.commits at gmail.com  Wed May 17 11:47:58 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 17 May 2017 08:47:58 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: turn GCBuffer into a base
 class again :(, so that we will be able to provide default methods
Message-ID: <591c70ae.53341c0a.3d8a0.50f6@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91322:7538ecbcf8b5
Date: 2017-05-17 17:42 +0200
http://bitbucket.org/pypy/pypy/changeset/7538ecbcf8b5/

Log:	turn GCBuffer into a base class again :(, so that we will be able to
	provide default methods

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -8,7 +8,7 @@
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
                               nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import (Buffer, GCBuffer,
+from rpython.rlib.buffer import (GCBuffer,
                                  get_gc_data_for_list_of_chars,
                                  get_gc_data_offset_for_list_of_chars)
 from pypy.interpreter.baseobjspace import W_Root
@@ -1258,8 +1258,8 @@
         start += step
 
 
- at GCBuffer
-class BytearrayBuffer(Buffer):
+ at GCBuffer.decorate
+class BytearrayBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, ba, readonly=False):
@@ -1289,7 +1289,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return GCBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -32,9 +32,8 @@
     Base class for buffers of bytes.
 
     Most probably, you do NOT want to use this as a lone base class, but
-    either inherit from RawBuffer or use the GCBuffer class decorator, so that
-    you automatically get the proper implementation of typed_read and
-    typed_write.
+    either inherit from RawBuffer or GCBuffer, so that you automatically get
+    the proper implementation of typed_read and typed_write.
     """
     _attrs_ = ['readonly']
     _immutable_ = True
@@ -133,13 +132,48 @@
         return llop.raw_store(lltype.Void, ptr, byte_offset, value)
 
 
-def GCBuffer(buffercls):
+class GCBuffer(Buffer):
     """
-    class decorator for a buffer which is baked by a GC-managed memory
-    area. It implements typed_read and typed_write in terms of
-    llop.gc_load_indexed and llop.gc_store_indexed.
+    Base class for a buffer which is baked by a GC-managed memory area. You
+    MUST also decorate the class with @GCBuffer.decorate: it implements
+    typed_read and typed_write in terms of llop.gc_load_indexed and
+    llop.gc_store_indexed.
+    """
+    _attrs_ = ['readonly', 'value']
+    _immutable_ = True
 
-    The target class MUST provide the following methods:
+    @staticmethod
+    def decorate(targetcls):
+        """
+        Create and attach specialized versions of typed_{read,write}. We need to
+        do this becase the JIT codewriters mandates that base_ofs is an
+        RPython constant.
+        """
+        if targetcls.__bases__ != (GCBuffer,):
+            raise ValueError("@GCBuffer.decorate should be used only on "
+                             "GCBuffer subclasses")
+
+        base_ofs = targetcls._get_gc_data_offset()
+        scale_factor = llmemory.sizeof(lltype.Char)
+
+        @specialize.ll_and_arg(1)
+        def typed_read(self, TP, byte_offset):
+            lldata = self._get_gc_data()
+            return llop.gc_load_indexed(TP, lldata, byte_offset,
+                                        scale_factor, base_ofs)
+
+        @specialize.ll_and_arg(1)
+        def typed_write(self, TP, byte_offset, value):
+            if self.readonly:
+                raise CannotWrite
+            lldata = self._get_gc_data()
+            value = lltype.cast_primitive(TP, value)
+            return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
+                                         scale_factor, base_ofs)
+
+        targetcls.typed_read = typed_read
+        targetcls.typed_write = typed_write
+        return targetcls
 
     @staticmethod
     def _get_gc_data_offset(self):
@@ -147,31 +181,16 @@
 
     def _get_gc_data(self):
         raise NotImplementedError
-    """
-    # We had to write this as a class decorator instead of a base class
-    # because we need to produce specialized versions of typed_{read,write} in
-    # which base_ofs is an RPython constant.
-    base_ofs = buffercls._get_gc_data_offset()
-    scale_factor = llmemory.sizeof(lltype.Char)
 
     @specialize.ll_and_arg(1)
     def typed_read(self, TP, byte_offset):
-        lldata = self._get_gc_data()
-        return llop.gc_load_indexed(TP, lldata, byte_offset,
-                                    scale_factor, base_ofs)
+        raise NotImplementedError("You MUST decorate this class with "
+                                  "@GCBuffer.decorate")
 
     @specialize.ll_and_arg(1)
     def typed_write(self, TP, byte_offset, value):
-        if self.readonly:
-            raise CannotWrite
-        lldata = self._get_gc_data()
-        value = lltype.cast_primitive(TP, value)
-        return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
-                                     scale_factor, base_ofs)
-
-    buffercls.typed_read = typed_read
-    buffercls.typed_write = typed_write
-    return buffercls
+        raise NotImplementedError("You MUST decorate this class with "
+                                  "@GCBuffer.decorate")
 
 
 def get_gc_data_for_list_of_chars(data):
@@ -184,8 +203,8 @@
     return llmemory.itemoffsetof(LIST.items.TO, 0)
 
 
- at GCBuffer
-class ByteBuffer(Buffer):
+ at GCBuffer.decorate
+class ByteBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, n):
@@ -212,8 +231,8 @@
         return get_gc_data_offset_for_list_of_chars()
 
 
- at GCBuffer
-class StringBuffer(Buffer):
+ at GCBuffer.decorate
+class StringBuffer(GCBuffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
 
diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
--- a/rpython/rlib/mutbuffer.py
+++ b/rpython/rlib/mutbuffer.py
@@ -3,11 +3,11 @@
 from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
 from rpython.rtyper.annlowlevel import llstr, hlstr
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.buffer import Buffer, GCBuffer
+from rpython.rlib.buffer import GCBuffer
 from rpython.rlib import jit
 
- at GCBuffer
-class MutableStringBuffer(Buffer):
+ at GCBuffer.decorate
+class MutableStringBuffer(GCBuffer):
     """
     A writeable buffer to incrementally fill a string of a fixed size.
 

From pypy.commits at gmail.com  Wed May 17 11:48:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 17 May 2017 08:48:01 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix
 test_pack_into_bytearray_delete
Message-ID: <591c70b1.03541c0a.7133f.1a0f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91323:1d430f535203
Date: 2017-05-17 17:47 +0200
http://bitbucket.org/pypy/pypy/changeset/1d430f535203/

Log:	fix test_pack_into_bytearray_delete

diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -1308,6 +1308,9 @@
     def _get_gc_data_offset():
         return get_gc_data_offset_for_list_of_chars()
 
+    def _get_gc_data_extra_offset(self):
+        return self.ba._offset
+
     def _get_gc_data(self):
         return get_gc_data_for_list_of_chars(self.ba._data)
 
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -159,6 +159,7 @@
         @specialize.ll_and_arg(1)
         def typed_read(self, TP, byte_offset):
             lldata = self._get_gc_data()
+            byte_offset += self._get_gc_data_extra_offset()
             return llop.gc_load_indexed(TP, lldata, byte_offset,
                                         scale_factor, base_ofs)
 
@@ -167,6 +168,7 @@
             if self.readonly:
                 raise CannotWrite
             lldata = self._get_gc_data()
+            byte_offset += self._get_gc_data_extra_offset()
             value = lltype.cast_primitive(TP, value)
             return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
                                          scale_factor, base_ofs)
@@ -179,6 +181,9 @@
     def _get_gc_data_offset(self):
         raise NotImplementedError
 
+    def _get_gc_data_extra_offset(self):
+        return 0
+
     def _get_gc_data(self):
         raise NotImplementedError
 

From pypy.commits at gmail.com  Wed May 17 11:47:56 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 17 May 2017 08:47:56 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: add a failing test,
 which is triggered when W_BytearrayObject._offset > 0
Message-ID: <591c70ac.56691c0a.fd0a0.eaa1@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91321:b207c44e74ae
Date: 2017-05-17 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/b207c44e74ae/

Log:	WIP: add a failing test, which is triggered when
	W_BytearrayObject._offset > 0

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -541,3 +541,12 @@
         buf = bytearray(len(expected))
         self.struct.pack_into("xxi", buf, 0, 42)
         assert buf == expected
+
+    def test_pack_into_bytearray_delete(self):
+        expected = self.struct.pack("i", 42)
+        # force W_BytearrayObject._delete_from_start
+        buf = bytearray(64)
+        del buf[:8]
+        self.struct.pack_into("i", buf, 0, 42)
+        buf = buf[:len(expected)]
+        assert buf == expected
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1111,7 +1111,7 @@
         list.__delitem__(self, index)
 
     def __getslice__(self, i, j):
-        return list.__getslice__(self.__as_list(), i, j)
+        return self.__class__(list.__getslice__(self.__as_list(), i, j))
 
     def __setslice__(self, i, j, new):
         lst = self.__as_list()
diff --git a/rpython/rlib/test/test_rgc.py b/rpython/rlib/test/test_rgc.py
--- a/rpython/rlib/test/test_rgc.py
+++ b/rpython/rlib/test/test_rgc.py
@@ -304,6 +304,30 @@
     data = subprocess.check_output([str(exename), '.', '.', '.'])
     assert data.strip().endswith('OK!')
 
+
+def test_nonmoving_raw_ptr_for_resizable_list_getslice():
+    def f(n):
+        lst = ['a', 'b', 'c', 'd', 'e']
+        lst = rgc.resizable_list_supporting_raw_ptr(lst)
+        lst = lst[:3]
+        lst.append(chr(n))
+        assert lst[3] == chr(n)
+        assert lst[-1] == chr(n)
+        #
+        ptr = rgc.nonmoving_raw_ptr_for_resizable_list(lst)
+        assert lst[:] == ['a', 'b', 'c', chr(n)]
+        assert lltype.typeOf(ptr) == rffi.CCHARP
+        assert [ptr[i] for i in range(4)] == ['a', 'b', 'c', chr(n)]
+        return lst
+    #
+    # direct untranslated run
+    lst = f(35)
+    assert isinstance(lst, rgc._ResizableListSupportingRawPtr)
+    #
+    # llinterp run
+    interpret(f, [35])
+
+
 def test_ll_for_resizable_list():
     def f(n):
         lst = ['a', 'b', 'c']

From pypy.commits at gmail.com  Wed May 17 14:12:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 17 May 2017 11:12:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Run sys.__interactivehook__() also when
 using -i
Message-ID: <591c92a3.91811c0a.d3945.d62f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91324:ae338b18eb22
Date: 2017-05-17 19:12 +0100
http://bitbucket.org/pypy/pypy/changeset/ae338b18eb22/

Log:	Run sys.__interactivehook__() also when using -i

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -694,8 +694,6 @@
                             del mainmodule.__file__
                         except (AttributeError, TypeError):
                             pass
-                if hasattr(sys, '__interactivehook__'):
-                    run_toplevel(sys.__interactivehook__)
                 # Then we need a prompt.
                 inspect = True
             else:
@@ -791,6 +789,8 @@
     if inspect_requested():
         try:
             from _pypy_interact import interactive_console
+            if hasattr(sys, '__interactivehook__'):
+                run_toplevel(sys.__interactivehook__)
             pypy_version_info = getattr(sys, 'pypy_version_info', sys.version_info)
             irc_topic = pypy_version_info[3] != 'final' or (
                             readenv and os.getenv('PYPY_IRC_TOPIC'))

From pypy.commits at gmail.com  Wed May 17 14:19:47 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 17 May 2017 11:19:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Disable enablerlcompleter() since it's
 broken on PyPy (note that virtualenv's site.py doesn't use it either)
Message-ID: <591c9443.928ddf0a.c053f.52eb@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91325:96d6c8419ada
Date: 2017-05-17 19:19 +0100
http://bitbucket.org/pypy/pypy/changeset/96d6c8419ada/

Log:	Disable enablerlcompleter() since it's broken on PyPy (note that
	virtualenv's site.py doesn't use it either)

diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -560,7 +560,8 @@
     setquit()
     setcopyright()
     sethelper()
-    enablerlcompleter()
+    # Disabled on PyPy: incompatible with our readline
+    #enablerlcompleter()
     aliasmbcs()
     execsitecustomize()
     if ENABLE_USER_SITE:

From pypy.commits at gmail.com  Wed May 17 14:42:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 17 May 2017 11:42:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Reenable the parts of enablerlcompleter()
 that do work (thanks o11c!)
Message-ID: <591c998a.6c8edf0a.dd814.7828@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91326:3f4f47fa82af
Date: 2017-05-17 19:40 +0100
http://bitbucket.org/pypy/pypy/changeset/3f4f47fa82af/

Log:	Reenable the parts of enablerlcompleter() that do work (thanks
	o11c!)

diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -408,7 +408,9 @@
             readline.parse_and_bind('tab: complete')
 
         try:
-            readline.read_init_file()
+            # Unimplemented on PyPy
+            #readline.read_init_file()
+            pass
         except OSError:
             # An OSError here could have many causes, but the most likely one
             # is that there's no .inputrc file (or .editrc file in the case of
@@ -560,8 +562,7 @@
     setquit()
     setcopyright()
     sethelper()
-    # Disabled on PyPy: incompatible with our readline
-    #enablerlcompleter()
+    enablerlcompleter()
     aliasmbcs()
     execsitecustomize()
     if ENABLE_USER_SITE:

From pypy.commits at gmail.com  Thu May 18 09:12:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 18 May 2017 06:12:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak for cython
Message-ID: <591d9da6.020a1c0a.466d0.290f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91327:6fcd279aca27
Date: 2017-05-18 16:11 +0300
http://bitbucket.org/pypy/pypy/changeset/6fcd279aca27/

Log:	tweak for cython

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -677,7 +677,7 @@
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
-                      PyClassObject'''.split():
+                      PyClassObject PyBaseExceptionObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -520,7 +520,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+ at cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error

From pypy.commits at gmail.com  Thu May 18 10:36:28 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 18 May 2017 07:36:28 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-obj-stealing: merge default into branch
Message-ID: <591db16c.86871c0a.0913.5ad6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-obj-stealing
Changeset: r91328:92c3ef911221
Date: 2017-05-18 17:35 +0300
http://bitbucket.org/pypy/pypy/changeset/92c3ef911221/

Log:	merge default into branch

diff too long, truncating to 2000 out of 4418 lines

diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -234,6 +234,9 @@
         if ('_abstract_' in cls.__dict__ or cls is Structure 
                                          or cls is union.Union):
             raise TypeError("abstract class")
+        if hasattr(cls, '_swappedbytes_'):
+            raise NotImplementedError("missing in PyPy: structure/union with "
+                                      "swapped (non-native) byte ordering")
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -79,10 +79,20 @@
 BOOL WINAPI CreateProcessA(char *, char *, void *,
                            void *, BOOL, DWORD, char *,
                            char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
+                           void *, BOOL, DWORD, wchar_t *,
+                           wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS     0x0001
+#define SEM_NOGPFAULTERRORBOX      0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX     0x8000
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
 )
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -42,8 +42,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -501,7 +501,14 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -34,3 +34,8 @@
 .. branch: controller-refactor
 
 Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (for post-5.7.1 builds) with sha256 checksum 
-``f1510452293f22e84d6059464e11f4c62ffd0e2ee97a52be9195bec8a70c6dce`` or
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -128,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -236,6 +236,9 @@
     copy out32\*.lib <somewhere in LIB>
     xcopy /S include\openssl <somewhere in INCLUDE>
 
+For tests you will also need the dlls::
+    nmake -f ms\ntdll.mak install
+    copy out32dll\*.dll <somewhere in PATH>
 
 TkInter module support
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -245,18 +248,17 @@
 directory found for the release script, create the dlls, libs, headers and
 runtime by running::
 
-	svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
-	svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
-	cd tcl85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
-	nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
-	cd ..\..\tk85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+    svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+    svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+    cd tcl85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
+    nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+    cd ..\..\tk85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+    copy ..\..\tcltk\bin\* <somewhere in PATH>
+    copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+    xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
 
 The lzma compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -341,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,9 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.buffer import StringBuffer
 
+from pypy.interpreter.buffer import BufferInterfaceNotFound
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -221,7 +223,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
 
@@ -233,7 +236,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -245,7 +249,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -254,7 +259,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -392,9 +398,6 @@
 class DescrMismatch(Exception):
     pass
 
-class BufferInterfaceNotFound(Exception):
-    pass
-
 @specialize.memo()
 def wrappable_class_name(Class):
     try:
@@ -1501,18 +1504,28 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+        except OperationError:
+            self._getarg_error("convertible to a buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.readbuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a readable buffer object")
+            self._getarg_error("convertible to a buffer", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+        except OperationError:
+            self._getarg_error("read-write buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.writebuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a writeable buffer object")
+            self._getarg_error("read-write buffer", w_obj)
 
     def charbuf_w(self, w_obj):
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
@@ -1541,12 +1554,10 @@
             if self.isinstance_w(w_obj, self.w_unicode):
                 return self.str(w_obj).readbuf_w(self)
             try:
-                return w_obj.buffer_w(self, 0)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.readbuf_w(self)
-            except BufferInterfaceNotFound:
+                return self.readbuf_w(w_obj)
+            except OperationError as e:
+                if not e.match(self, self.w_TypeError):
+                    raise
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_bytes):
@@ -1558,16 +1569,7 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
-            try:
-                return w_obj.buffer_w(self, self.BUF_WRITABLE)
-            except OperationError:
-                self._getarg_error("read-write buffer", w_obj)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.writebuf_w(self)
-            except BufferInterfaceNotFound:
-                self._getarg_error("read-write buffer", w_obj)
+            return self.writebuf_w(w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
@@ -1654,6 +1656,23 @@
     def fsencode_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
 
+    def byte_w(self, w_obj):
+        """
+        Convert an index-like object to an interp-level char
+
+        Used for app-level code like "bytearray(b'abc')[0] = 42".
+        """
+        if self.isinstance_w(w_obj, self.w_bytes):
+            string = self.bytes_w(w_obj)
+            if len(string) != 1:
+                raise oefmt(self.w_ValueError, "string must be of size 1")
+            return string[0]
+        value = self.getindex_w(w_obj, None)
+        if not 0 <= value < 256:
+            # this includes the OverflowError in case the long is too large
+            raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
+        return chr(value)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,295 @@
+from rpython.rlib.buffer import StringBuffer, SubBuffer
+
+from pypy.interpreter.error import oefmt
+
+class BufferInterfaceNotFound(Exception):
+    pass
+
+
+class BufferView(object):
+    """Abstract base class for buffers."""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        return ''.join(self._copy_buffer())
+
+    def getbytes(self, start, size):
+        """Return `size` bytes starting at byte offset `start`.
+
+        This is a low-level operation, it is up to the caller to ensure that
+        the data requested actually correspond to items accessible from the
+        BufferView.
+        Note that `start` may be negative, e.g. if the buffer is reversed.
+        """
+        raise NotImplementedError
+
+    def setbytes(self, start, string):
+        raise NotImplementedError
+
+    def get_raw_address(self):
+        raise ValueError("no raw buffer")
+
+    def as_readbuf(self):
+        # Inefficient. May be overridden.
+        return StringBuffer(self.as_str())
+
+    def as_writebuf(self):
+        """Return a writable Buffer sharing the same data as `self`."""
+        raise BufferInterfaceNotFound
+
+    def getformat(self):
+        raise NotImplementedError
+
+    def getitemsize(self):
+        raise NotImplementedError
+
+    def getndim(self):
+        raise NotImplementedError
+
+    def getshape(self):
+        raise NotImplementedError
+
+    def getstrides(self):
+        raise NotImplementedError
+
+    def releasebuffer(self):
+        pass
+
+    def value_from_bytes(self, space, s):
+        from pypy.module.struct.formatiterator import UnpackFormatIterator
+        buf = StringBuffer(s)
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(self.getformat())
+        return fmtiter.result_w[0]
+
+    def _copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.getbytes(0, itemsize)]
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim + 1, data, off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        bytesize = self.getlength()
+        copiedbytes = 0
+        for i in range(step):
+            bytes = self.getbytes(off, itemsize)
+            data.append(bytes)
+            copiedbytes += len(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if copiedbytes >= bytesize:
+                break
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        shape = self.getshape()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        # TODO suboffsets?
+        strides = self.getstrides()
+        return strides[dim] * index
+
+    def w_getitem(self, space, idx):
+        offset = self.get_offset(space, 0, idx)
+        itemsize = self.getitemsize()
+        # TODO: this probably isn't very fast
+        data = self.getbytes(offset, itemsize)
+        return space.newbytes(data)
+
+    def new_slice(self, start, step, slicelength):
+        return BufferSlice(self, start, step, slicelength)
+
+    def w_tolist(self, space):
+        dim = self.getndim()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            n = self.getshape()[0]
+            values_w = [space.ord(self.w_getitem(space, i)) for i in range(n)]
+            return space.newlist(values_w)
+        else:
+            return self._tolist_rec(space, 0, 0)
+
+    def _tolist_rec(self, space, start, idim):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim + 1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            values_w = [
+                self.value_from_bytes(space, self.getbytes(pos, itemsize))
+                for pos in range(start, start + bytecount, stride)]
+            return space.newlist(values_w)
+
+        items = [None] * dimshape
+        for i in range(dimshape):
+            item = self._tolist_rec(space, start, idim + 1)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+    def wrap(self, space):
+        return space.newmemoryview(self)
+
+
+class SimpleView(BufferView):
+    _attrs_ = ['readonly', 'data']
+    _immutable_ = True
+
+    def __init__(self, data):
+        self.data = data
+        self.readonly = self.data.readonly
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def as_str(self):
+        return self.data.as_str()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def setbytes(self, offset, s):
+        self.data.setslice(offset, s)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_readbuf(self):
+        return self.data
+
+    def as_writebuf(self):
+        assert not self.data.readonly
+        return self.data
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        assert dim == 0
+        nitems = self.getlength()
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        return index
+
+    def w_getitem(self, space, idx):
+        idx = self.get_offset(space, 0, idx)
+        ch = self.data[idx]
+        return space.newbytes(ch)
+
+    def new_slice(self, start, step, slicelength):
+        if step == 1:
+            return SimpleView(SubBuffer(self.data, start, slicelength))
+        else:
+            return BufferSlice(self, start, step, slicelength)
+
+
+class BufferSlice(BufferView):
+    _immutable_ = True
+    _attrs_ = ['parent', 'readonly', 'shape', 'strides', 'start', 'step']
+
+    def __init__(self, parent, start, step, length):
+        self.parent = parent
+        self.readonly = self.parent.readonly
+        self.strides = parent.getstrides()[:]
+        self.start = start
+        self.step = step
+        self.strides[0] *= step
+        self.shape = parent.getshape()[:]
+        self.shape[0] = length
+
+    def getlength(self):
+        return self.shape[0] * self.getitemsize()
+
+    def getbytes(self, start, size):
+        offset = self.start * self.parent.getstrides()[0]
+        return self.parent.getbytes(offset + start, size)
+
+    def setbytes(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        offset = self.start * self.parent.getstrides()[0]
+        self.parent.setbytes(offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        offset = self.start * self.parent.getstrides()[0]
+        return rffi.ptradd(self.parent.get_raw_address(), offset)
+
+    def getformat(self):
+        return self.parent.getformat()
+
+    def getitemsize(self):
+        return self.parent.getitemsize()
+
+    def getndim(self):
+        return self.parent.getndim()
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides
+
+    def parent_index(self, idx):
+        return self.start + self.step * idx
+
+    def w_getitem(self, space, idx):
+        return self.parent.w_getitem(space, self.parent_index(idx))
+
+    def new_slice(self, start, step, slicelength):
+        real_start = start + self.start
+        real_step = self.step * step
+        return BufferSlice(self.parent, real_start, real_step, slicelength)
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,5 +1,6 @@
 class AppTestCompile:
     def test_simple(self):
+        import sys
         co = compile('1+2', '?', 'eval')
         assert eval(co) == 3
         co = compile(buffer('1+2'), '?', 'eval')
@@ -8,8 +9,10 @@
         assert str(exc.value) == "compile() expected string without null bytes"
         exc = raises(TypeError, compile, unichr(0), '?', 'eval')
         assert str(exc.value) == "compile() expected string without null bytes"
-        exc = raises(TypeError, compile, memoryview('1+2'), '?', 'eval')
-        assert str(exc.value) == "expected a readable buffer object"
+
+        if '__pypy__' in sys.modules:
+            co = compile(memoryview('1+2'), '?', 'eval')
+            assert eval(co) == 3
         compile("from __future__ import with_statement", "<test>", "exec")
         raises(SyntaxError, compile, '-', '?', 'eval')
         raises(ValueError, compile, '"\\xt"', '?', 'eval')
@@ -50,7 +53,8 @@
         co1 = compile('print 1', '<string>', 'exec', _ast.PyCF_ONLY_AST)
         raises(TypeError, compile, co1, '<ast>', 'eval')
         co2 = compile('1+1', '<string>', 'eval', _ast.PyCF_ONLY_AST)
-        compile(co2, '<ast>', 'eval')
+        tree = compile(co2, '<ast>', 'eval')
+        assert compile(co2, '<ast>', 'eval', _ast.PyCF_ONLY_AST) is co2
 
     def test_leading_newlines(self):
         src = """
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -4,6 +4,7 @@
 from pypy.module._cffi_backend import cdataobj, ctypeptr, ctypearray
 from pypy.module._cffi_backend import ctypestruct
 from pypy.objspace.std.bufferobject import W_Buffer
+from pypy.interpreter.buffer import SimpleView
 
 from rpython.rlib.buffer import Buffer
 from rpython.rtyper.annlowlevel import llstr
@@ -60,7 +61,7 @@
         if space.isinstance_w(w_other, space.w_unicode):
             return space.w_NotImplemented
         try:
-            other_buf = space.buffer_w(w_other, space.BUF_SIMPLE)
+            other_buf = space.readbuf_w(w_other)
         except OperationError as e:
             if e.async(space):
                 raise
diff --git a/pypy/module/_cffi_backend/func.py b/pypy/module/_cffi_backend/func.py
--- a/pypy/module/_cffi_backend/func.py
+++ b/pypy/module/_cffi_backend/func.py
@@ -105,24 +105,10 @@
 # ____________________________________________________________
 
 def _fetch_as_read_buffer(space, w_x):
-    # xxx do we really need to implement the same mess as in CPython 2.7
-    # w.r.t. buffers and memoryviews??
-    try:
-        buf = space.readbuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_SIMPLE)
-    return buf
+    return space.readbuf_w(w_x)
 
 def _fetch_as_write_buffer(space, w_x):
-    try:
-        buf = space.writebuf_w(w_x)
-    except OperationError as e:
-        if not e.match(space, space.w_TypeError):
-            raise
-        buf = space.buffer_w(w_x, space.BUF_WRITABLE)
-    return buf
+    return space.writebuf_w(w_x)
 
 @unwrap_spec(w_ctype=ctypeobj.W_CType)
 def from_buffer(space, w_ctype, w_x):
diff --git a/pypy/module/_codecs/test/test_codecs.py b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -292,8 +292,8 @@
             assert bytes2.decode("unicode_internal") == u"\U00010098"
         assert bytes.decode("unicode_internal") == u"a"
         assert _codecs.unicode_internal_decode(array.array('c', bytes))[0] == u"a"
-        exc = raises(TypeError, _codecs.unicode_internal_decode, memoryview(bytes))
-        assert str(exc.value) == "expected a readable buffer object"
+        if '__pypy__' in sys.modules:
+            assert _codecs.unicode_internal_decode(memoryview(bytes))[0] == u"a"
 
     def test_raw_unicode_escape(self):
         assert unicode("\u0663", "raw-unicode-escape") == u"\u0663"
diff --git a/pypy/module/_io/interp_bufferedio.py b/pypy/module/_io/interp_bufferedio.py
--- a/pypy/module/_io/interp_bufferedio.py
+++ b/pypy/module/_io/interp_bufferedio.py
@@ -1,12 +1,15 @@
 from __future__ import with_statement
 
+from rpython.rlib.signature import signature
+from rpython.rlib import types
+
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import (
     TypeDef, GetSetProperty, generic_new_descr, interp_attrproperty_w)
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rgc import (
-    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
-from rpython.rlib.buffer import Buffer
+from pypy.interpreter.buffer import SimpleView
+
+from rpython.rlib.buffer import ByteBuffer, SubBuffer
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rarithmetic import r_longlong, intmask
 from rpython.rlib import rposix
@@ -16,7 +19,6 @@
     check_readable_w, check_writable_w, check_seekable_w)
 from pypy.module._io.interp_io import W_BlockingIOError
 from rpython.rlib import rthread
-from rpython.rtyper.lltypesystem import rffi
 
 STATE_ZERO, STATE_OK, STATE_DETACHED = range(3)
 
@@ -88,12 +90,16 @@
         self._unsupportedoperation(space, "detach")
 
     def readinto_w(self, space, w_buffer):
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        return self._readinto(space, w_buffer, "read")
+
+    def _readinto(self, space, w_buffer, methodname):
+        rwbuffer = space.writebuf_w(w_buffer)
         length = rwbuffer.getlength()
-        w_data = space.call_method(self, "read", space.newint(length))
+        w_data = space.call_method(self, methodname, space.newint(length))
 
         if not space.isinstance_w(w_data, space.w_bytes):
-            raise oefmt(space.w_TypeError, "read() should return bytes")
+            raise oefmt(space.w_TypeError, "%s() should return bytes",
+                        methodname)
         data = space.bytes_w(w_data)
         rwbuffer.setslice(0, data)
         return space.newint(len(data))
@@ -108,25 +114,6 @@
     readinto = interp2app(W_BufferedIOBase.readinto_w),
 )
 
-class RawBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, buf, start, length):
-        self.buf = buf
-        self.start = start
-        self.length = length
-        self.readonly = False
-
-    def getlength(self):
-        return self.length
-
-    def setitem(self, index, char):
-        self.buf[self.start + index] = char
-
-    def get_raw_address(self):
-        ptr = nonmoving_raw_ptr_for_resizable_list(self.buf)
-        return rffi.ptradd(ptr, self.start)
-
 class BufferedMixin:
     _mixin_ = True
 
@@ -165,8 +152,7 @@
             raise oefmt(space.w_ValueError,
                         "buffer size must be strictly positive")
 
-        self.buffer = resizable_list_supporting_raw_ptr(['\0'] *
-                                                        self.buffer_size)
+        self.buffer = ByteBuffer(self.buffer_size)
 
         self.lock = TryLock(space)
 
@@ -238,6 +224,7 @@
 
     # ______________________________________________
 
+    @signature(types.any(), returns=types.int())
     def _readahead(self):
         if self.readable and self.read_end != -1:
             available = self.read_end - self.pos
@@ -278,7 +265,7 @@
                 else:
                     offset = pos
                 if -self.pos <= offset <= available:
-                    newpos = self.pos + offset
+                    newpos = self.pos + int(offset)
                     assert newpos >= 0
                     self.pos = newpos
                     return space.newint(current - available + offset)
@@ -374,11 +361,7 @@
         return written
 
     def _raw_write(self, space, start, end):
-        # XXX inefficient
-        l = []
-        for i in range(start, end):
-            l.append(self.buffer[i])
-        return self._write(space, ''.join(l))
+        return self._write(space, self.buffer[start:end])
 
     def detach_w(self, space):
         self._check_init(space)
@@ -428,6 +411,7 @@
     @unwrap_spec(size=int)
     def peek_w(self, space, size=0):
         self._check_init(space)
+        self._check_closed(space, "peek of closed file")
         with self.lock:
             if self.writable:
                 self._flush_and_rewind_unlocked(space)
@@ -439,7 +423,7 @@
             # buffer.
             have = self._readahead()
             if have > 0:
-                data = ''.join(self.buffer[self.pos:self.pos+have])
+                data = self.buffer[self.pos:self.pos+have]
                 return space.newbytes(data)
 
             # Fill the buffer from the raw stream, and copy it to the result
@@ -449,7 +433,7 @@
             except BlockingIOError:
                 size = 0
             self.pos = 0
-            data = ''.join(self.buffer[:size])
+            data = self.buffer[0:size]
             return space.newbytes(data)
 
     @unwrap_spec(size=int)
@@ -486,7 +470,7 @@
             if size > have:
                 size = have
             endpos = self.pos + size
-            data = ''.join(self.buffer[self.pos:endpos])
+            data = self.buffer[self.pos:endpos]
             self.pos = endpos
             return space.newbytes(data)
 
@@ -498,7 +482,7 @@
         current_size = self._readahead()
         data = None
         if current_size:
-            data = ''.join(self.buffer[self.pos:self.pos + current_size])
+            data = self.buffer[self.pos:self.pos + current_size]
             builder.append(data)
             self.pos += current_size
         # We're going past the buffer's bounds, flush it
@@ -524,11 +508,13 @@
         return space.newbytes(builder.build())
 
     def _raw_read(self, space, buffer, start, length):
+        assert buffer is not None
         length = intmask(length)
-        w_buf = space.newbuffer(RawBuffer(buffer, start, length))
+        start = intmask(start)
+        w_view = SimpleView(SubBuffer(buffer, start, length)).wrap(space)
         while True:
             try:
-                w_size = space.call_method(self.w_raw, "readinto", w_buf)
+                w_size = space.call_method(self.w_raw, "readinto", w_view)
             except OperationError as e:
                 if trap_eintr(space, e):
                     continue  # try again
@@ -565,12 +551,12 @@
         if n <= current_size:
             return self._read_fast(n)
 
-        result_buffer = resizable_list_supporting_raw_ptr(['\0'] * n)
+        result_buffer = ByteBuffer(n)
         remaining = n
         written = 0
         if current_size:
-            for i in range(current_size):
-                result_buffer[written + i] = self.buffer[self.pos + i]
+            result_buffer.setslice(
+                written, self.buffer[self.pos:self.pos + current_size])
             remaining -= current_size
             written += current_size
             self.pos += current_size
@@ -592,7 +578,7 @@
                     return None
                 size = 0
             if size == 0:
-                return ''.join(result_buffer[:written])
+                return result_buffer[0:written]
             remaining -= size
             written += size
 
@@ -614,14 +600,13 @@
             if remaining > 0:
                 if size > remaining:
                     size = remaining
-                for i in range(size):
-                    result_buffer[written + i] = self.buffer[self.pos + i]
+                result_buffer.setslice(
+                    written, self.buffer[self.pos:self.pos + size])
                 self.pos += size
-
                 written += size
                 remaining -= size
 
-        return ''.join(result_buffer[:written])
+        return result_buffer[0:written]
 
     def _read_fast(self, n):
         """Read n bytes from the buffer if it can, otherwise return None.
@@ -629,7 +614,7 @@
         current_size = self._readahead()
         if n <= current_size:
             endpos = self.pos + n
-            res = ''.join(self.buffer[self.pos:endpos])
+            res = self.buffer[self.pos:endpos]
             self.pos = endpos
             return res
         return None
@@ -652,11 +637,11 @@
         else:
             pos = -1
         if pos >= 0:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:pos+1]))
+            w_res = space.newbytes(self.buffer[self.pos:pos+1])
             self.pos = pos + 1
             return w_res
         if have == limit:
-            w_res = space.newbytes(''.join(self.buffer[self.pos:self.pos+have]))
+            w_res = space.newbytes(self.buffer[self.pos:self.pos+have])
             self.pos += have
             return w_res
 
@@ -665,7 +650,7 @@
             # Now we try to get some more from the raw stream
             chunks = []
             if have > 0:
-                chunks.extend(self.buffer[self.pos:self.pos+have])
+                chunks.append(self.buffer[self.pos:self.pos+have])
                 written += have
                 self.pos += have
                 if limit >= 0:
@@ -683,13 +668,14 @@
                 pos = 0
                 found = False
                 while pos < have:
-                    c = self.buffer[pos]
+                    # 'buffer.data[]' instead of 'buffer[]' because RPython...
+                    c = self.buffer.data[pos]
                     pos += 1
                     if c == '\n':
                         self.pos = pos
                         found = True
                         break
-                chunks.extend(self.buffer[0:pos])
+                chunks.append(self.buffer[0:pos])
                 if found:
                     break
                 if have == limit:
@@ -716,7 +702,6 @@
         size = len(data)
 
         with self.lock:
-
             if (not (self.readable and self.read_end != -1) and
                 not (self.writable and self.write_end != -1)):
                 self.pos = 0
@@ -746,7 +731,8 @@
                     self._reader_reset_buf()
                 # Make some place by shifting the buffer
                 for i in range(self.write_pos, self.write_end):
-                    self.buffer[i - self.write_pos] = self.buffer[i]
+                    # XXX: messing with buffer internals
+                    self.buffer.data[i - self.write_pos] = self.buffer.data[i]
                 self.write_end -= self.write_pos
                 self.raw_pos -= self.write_pos
                 newpos = self.pos - self.write_pos
diff --git a/pypy/module/_io/interp_bytesio.py b/pypy/module/_io/interp_bytesio.py
--- a/pypy/module/_io/interp_bytesio.py
+++ b/pypy/module/_io/interp_bytesio.py
@@ -34,17 +34,17 @@
         size = convert_size(space, w_size)
         return space.newbytes(self.read(size))
 
+    def read1_w(self, space, w_size):
+        return self.read_w(space, w_size)
+
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
         return space.newbytes(self.readline(limit))
 
-    def read1_w(self, space, w_size):
-        return self.read_w(space, w_size)
-
     def readinto_w(self, space, w_buffer):
         self._check_closed(space)
-        rwbuffer = space.getarg_w('w*', w_buffer)
+        rwbuffer = space.writebuf_w(w_buffer)
         size = rwbuffer.getlength()
 
         output = self.read(size)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,5 +1,6 @@
+from rpython.rtyper.lltypesystem import rffi
+
 from rpython.rlib.buffer import Buffer
-from rpython.rtyper.lltypesystem import rffi
 
 # XXX not the most efficient implementation
 
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -1,5 +1,6 @@
 import sys
 from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.buffer import SimpleView
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import interp_attrproperty
@@ -371,7 +372,7 @@
         self._ll_buffer = self.ll_buffer
 
     def buffer_w(self, space, flags):
-        return RawFFIBuffer(self)
+        return SimpleView(RawFFIBuffer(self))
 
     def readbuf_w(self, space):
         return RawFFIBuffer(self)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -677,7 +677,7 @@
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
-                      PyClassObject'''.split():
+                      PyClassObject PyBaseExceptionObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
 
     if w_dict is None:
         return 0
-
+    if not space.isinstance_w(w_dict, space.w_dict):
+        return 0 
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         Py_IncRef(space, py_dict.c__tmpkeys)
     else:
+        if not py_dict.c__tmpkeys:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
         w_keys = from_ref(space, py_dict.c__tmpkeys)
     ppos[0] += 1
     if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/memoryobject.py b/pypy/module/cpyext/memoryobject.py
--- a/pypy/module/cpyext/memoryobject.py
+++ b/pypy/module/cpyext/memoryobject.py
@@ -41,15 +41,15 @@
     assert isinstance(w_obj, W_MemoryView)
     py_obj = rffi.cast(PyMemoryViewObject, py_obj)
     view = py_obj.c_view
-    ndim = w_obj.buf.getndim()
+    ndim = w_obj.getndim()
     if ndim >= Py_MAX_NDIMS:
         # XXX warn?
         return
-    fill_Py_buffer(space, w_obj.buf, view)
+    fill_Py_buffer(space, w_obj.view, view)
     try:
-        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.buf.get_raw_address())
+        view.c_buf = rffi.cast(rffi.VOIDP, w_obj.view.get_raw_address())
         view.c_obj = make_ref(space, w_userdata)
-        rffi.setintfield(view, 'c_readonly', w_obj.buf.readonly)
+        rffi.setintfield(view, 'c_readonly', w_obj.view.readonly)
     except ValueError:
         w_s = w_obj.descr_tobytes(space)
         view.c_obj = make_ref(space, w_s)
@@ -95,7 +95,6 @@
     mem_obj.c_view.c_obj = rffi.cast(PyObject, 0)
     _dealloc(space, py_obj)
 
-
 def fill_Py_buffer(space, buf, view):
     # c_buf, c_obj have been filled in
     ndim = buf.getndim()
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,13 +14,15 @@
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext import userslot
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
+
 from rpython.rlib.buffer import Buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
@@ -263,7 +265,7 @@
     check_num_args(space, w_args, 1)
     args_w = space.fixedview(w_args)
     index = space.int_w(space.index(args_w[0]))
-    null = lltype.nullptr(PyObject.TO)
+    null = rffi.cast(PyObject, 0)
     res = generic_cpy_call(space, func_target, w_self, index, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
@@ -292,7 +294,8 @@
     func_target = rffi.cast(objobjargproc, func)
     check_num_args(space, w_args, 1)
     w_key, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, None)
+    null = rffi.cast(PyObject, 0)
+    res = generic_cpy_call(space, func_target, w_self, w_key, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.w_None
@@ -322,7 +325,7 @@
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.newint(res)
 
-class CPyBuffer(Buffer):
+class CPyBuffer(BufferView):
     # Similar to Py_buffer
     _immutable_ = True
 
@@ -333,9 +336,12 @@
         self.space = space
         self.ptr = ptr
         self.size = size
-        self.w_obj = w_obj # kept alive
+        self.w_obj = w_obj  # kept alive
         self.pyobj = as_pyobj(space, w_obj)
         self.format = format
+        self.ndim = ndim
+        self.itemsize = itemsize
+
         if not shape:
             self.shape = [size]
         else:
@@ -344,8 +350,6 @@
             self.strides = [1]
         else:
             self.strides = strides
-        self.ndim = ndim
-        self.itemsize = itemsize
         self.readonly = readonly
         self.needs_decref = needs_decref
         self.releasebufferproc = releasebufferproc
@@ -378,8 +382,20 @@
     def getlength(self):
         return self.size
 
-    def getitem(self, index):
-        return self.ptr[index]
+    def getbytes(self, start, size):
+        return ''.join([self.ptr[i] for i in range(start, start + size)])
+
+    def setbytes(self, start, string):
+        # absolutely no safety checks, what could go wrong?
+        for i in range(len(string)):
+            self.ptr[start + i] = string[i]
+
+    def as_readbuf(self):
+        return CBuffer(self)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return CBuffer(self)
 
     def get_raw_address(self):
         return rffi.cast(rffi.CCHARP, self.ptr)
@@ -399,10 +415,6 @@
     def getndim(self):
         return self.ndim
 
-    def setitem(self, index, char):
-        # absolutely no safety checks, what could go wrong?
-        self.ptr[index] = char
-
 class FQ(rgc.FinalizerQueue):
     Class = CPyBuffer
     def finalizer_trigger(self):
@@ -414,6 +426,37 @@
 
 fq = FQ()
 
+
+class CBuffer(Buffer):
+    _immutable_ = True
+    def __init__(self, view):
+        self.view = view
+        self.readonly = view.readonly
+
+    def getlength(self):
+        return self.view.getlength()
+
+    def getitem(self, index):
+        return self.view.ptr[index]
+
+    def getslice(self, start, stop, step, size):
+        assert step == 1
+        assert stop - start == size
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), start)
+        return rffi.charpsize2str(ptr, size)
+
+    def setitem(self, index, char):
+        self.view.ptr[index] = char
+
+    def setslice(self, index, s):
+        assert s is not None
+        ptr = rffi.ptradd(cts.cast('char *', self.view.ptr), index)
+        rffi.str2chararray(s, ptr, len(s))
+
+    def get_raw_address(self):
+        return cts.cast('char *', self.view.ptr)
+
+
 def wrap_getreadbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
     py_obj = make_ref(space, w_self)
@@ -427,10 +470,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self,
+        view = CPyBuffer(space, ptr[0], size, w_self,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getwritebuffer(space, w_self, w_args, func):
     func_target = rffi.cast(readbufferproc, func)
@@ -445,10 +488,10 @@
         size = generic_cpy_call(space, func_target, w_self, index, ptr)
         if size < 0:
             space.fromcache(State).check_and_raise_exception(always=True)
-        buf = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
+        view = CPyBuffer(space, ptr[0], size, w_self, readonly=False,
                                releasebufferproc=rbp)
-        fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        fq.register_finalizer(view)
+        return space.newbuffer(CBuffer(view))
 
 def wrap_getbuffer(space, w_self, w_args, func):
     func_target = rffi.cast(getbufferproc, func)
@@ -490,7 +533,7 @@
                             needs_decref=True,
                             releasebufferproc = rbp)
         fq.register_finalizer(buf)
-        return space.newbuffer(buf)
+        return buf.wrap(space)
 
 def get_richcmp_func(OP_CONST):
     def inner(space, w_self, w_args, func):
@@ -570,6 +613,8 @@
             handled = True
 
     for tp_name, attr in [('tp_hash', '__hash__'),
+                          ('tp_as_sequence.c_sq_length', '__len__'),
+                          ('tp_as_mapping.c_mp_length', '__len__'),
                          ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -595,7 +640,8 @@
                           ('tp_as_number.c_nb_xor', '__xor__'),
                           ('tp_as_number.c_nb_or', '__or__'),
                           ('tp_as_sequence.c_sq_concat', '__add__'),
-                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+                          ('tp_as_mapping.c_mp_subscript', '__getitem__'),
                           ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -609,7 +655,7 @@
             handled = True
 
     # binary-with-Py_ssize_t-type
-    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -638,7 +684,48 @@
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
             handled = True
+    # ternary-with-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
 
+            @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, w_arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, w_arg1)
+                return 0
+            handled = True
+    # ternary-Py_size_t-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
+
+            @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, space.newint(arg1))
+                return 0
+            handled = True
     if handled:
         pass
     elif name == 'tp_setattro':
@@ -667,6 +754,7 @@
         def slot_tp_getattro(space, w_self, w_name):
             return space.call_function(getattr_fn, w_self, w_name)
         slot_func = slot_tp_getattro
+
     elif name == 'tp_call':
         call_fn = w_type.getdictvalue(space, '__call__')
         if call_fn is None:
@@ -744,24 +832,24 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         args = Arguments(space, [space.newint(flags)])
         w_obj = space.call_args(space.get(buff_fn, w_self), args)
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
@@ -771,23 +859,23 @@
     @slot_function([PyObject, Py_bufferP, rffi.INT_real],
             rffi.INT_real, error=-1)
     @func_renamer("cpyext_%s_%s" % (name, typedef.name))
-    def buff_w(space, w_self, view, flags):
+    def buff_w(space, w_self, c_view, flags):
         w_obj = w_self
-        if view:
+        if c_view:
             #like PyObject_GetBuffer
             flags = widen(flags)
             buf = space.buffer_w(w_obj, flags)
             try:
-                view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
-                view.c_obj = make_ref(space, w_obj)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, buf.get_raw_address())
+                c_view.c_obj = make_ref(space, w_obj)
             except ValueError:
                 s = buf.as_str()
                 w_s = space.newbytes(s)
-                view.c_obj = make_ref(space, w_s)
-                view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
+                c_view.c_obj = make_ref(space, w_s)
+                c_view.c_buf = rffi.cast(rffi.VOIDP, rffi.str2charp(
                                         s, track_allocation=False))
-                rffi.setintfield(view, 'c_readonly', 1)
-            ret = fill_Py_buffer(space, buf, view)
+                rffi.setintfield(c_view, 'c_readonly', 1)
+            ret = fill_Py_buffer(space, buf, c_view)
             return ret
         return 0
     return buff_w
diff --git a/pypy/module/cpyext/test/buffer_test.c b/pypy/module/cpyext/test/buffer_test.c
--- a/pypy/module/cpyext/test/buffer_test.c
+++ b/pypy/module/cpyext/test/buffer_test.c
@@ -344,6 +344,7 @@
 #endif
     if (m == NULL)
         INITERROR;
+    PyMyArrayType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&PyMyArrayType) < 0)
         INITERROR;
     Py_INCREF(&PyMyArrayType);
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -255,4 +255,60 @@
             ])
         d = module.get_type_dict(1)
         assert d['real'].__get__(1, 1) == 1
-
+    def test_advanced(self):
+        module = self.import_extension('foo', [
+            ("dict_len", "METH_O",
+            '''
+                int ret = args->ob_type->tp_as_mapping->mp_length(args);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_setitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 3 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1),
+                        PyTuple_GetItem(args, 2));
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_delitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 2 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1), NULL);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_next", "METH_VARARGS",
+            '''
+                PyObject *key, *value;
+                PyObject *arg = NULL;
+                Py_ssize_t pos = 0;
+                int ret = 0;
+                if ((PyArg_ParseTuple(args, "|O", &arg))) {
+                    if (arg && PyDict_Check(arg)) {
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                        /* test no crash if pos is not reset to 0*/
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                    }
+                }
+                return PyLong_FromLong(ret);
+            '''),
+            ])
+        d = {'a': 1, 'b':2}
+        assert module.dict_len(d) == 2
+        assert module.dict_setitem(d, 'a', 'c') == 0
+        assert d['a'] == 'c'
+        assert module.dict_delitem(d, 'a') == 0
+        r = module.dict_next({'a': 1, 'b': 2})
+        assert r == 2
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -111,7 +111,7 @@
                 PyObject* obj = PyTuple_GetItem(args, 0);
                 PyObject* memoryview = PyMemoryView_FromObject(obj);
                 if (memoryview == NULL)
-                    return PyLong_FromLong(-1);
+                    return NULL;
                 view = PyMemoryView_GET_BUFFER(memoryview);
                 Py_DECREF(memoryview);
                 return PyLong_FromLong(view->len / view->itemsize);
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -47,6 +47,33 @@
         w_year = space.getattr(w_obj, space.newtext('year'))
         assert space.int_w(w_year) == 1
 
+    def test_descr_slots(self, space, api):
+        w_descr = space.appexec([], """():
+            class Descr(object):
+                def __get__(self, obj, type):
+                    return 42
+                def __set__(self, obj, value):
+                    obj.append('set')
+                def __delete__(self, obj):
+                    obj.append('del')
+            return Descr()
+            """)
+        w_descrtype = space.type(w_descr)
+        py_descr = make_ref(space, w_descr)
+        py_descrtype = rffi.cast(PyTypeObjectPtr, make_ref(space, w_descrtype))
+        w_obj = space.newlist([])
+        py_obj = make_ref(space, w_obj)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, py_obj, py_obj)
+        assert space.int_w(w_res) == 42
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, make_ref(space, space.w_None)) == 0
+        assert generic_cpy_call(
+            space, py_descrtype.c_tp_descr_set,
+            py_descr, py_obj, None) == 0
+        assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
         # to see that the functions are being used,
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -34,7 +34,7 @@
 cpython_struct("PyTupleObject", PyTupleObjectFields, PyTupleObjectStruct)
 
 @bootstrap_function
-def init_stringobject(space):
+def init_tupleobject(space):
     "Type description of PyTupleObject"
     make_typedescr(space.w_tuple.layout.typedef,
                    basestruct=PyTupleObject.TO,
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -520,7 +520,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+ at cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -109,4 +109,14 @@
 def slot_tp_getattr(space, w_obj1, w_obj2):
     return space.getattr(w_obj1, w_obj2)
 
+ at slot_function([PyObject, PyObject, PyObject], PyObject)
+def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    return space.get(w_self, w_obj, w_type)
 
+ at slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+def slot_tp_descr_set(space, w_self, w_obj, w_value):
+    if w_value is not None:
+        space.set(w_self, w_obj, w_value)
+    else:
+        space.delete(w_self, w_obj)
+    return 0
diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -920,90 +920,42 @@
 class W_GroupBy(W_Root):
     def __init__(self, space, w_iterable, w_fun):
         self.space = space
-        self.w_iterable = self.space.iter(w_iterable)
-        if space.is_none(w_fun):
-            self.w_fun = None
-        else:
-            self.w_fun = w_fun
-        self.index = 0
-        self.lookahead = False
-        self.exhausted = False
-        self.started = False
-        # new_group - new group not started yet, next should not skip any items
-        self.new_group = True
-        self.w_lookahead = self.space.w_None
-        self.w_key = self.space.w_None
+        self.w_iterator = self.space.iter(w_iterable)
+        if w_fun is None:
+            w_fun = space.w_None
+        self.w_keyfunc = w_fun
+        self.w_tgtkey = None
+        self.w_currkey = None
+        self.w_currvalue = None
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        self._skip_to_next_iteration_group()
+        w_key = self.w_tgtkey = self.w_currkey
+        w_grouper = W_GroupByIterator(self, w_key)
+        return self.space.newtuple([w_key, w_grouper])
 
-        if not self.new_group:
-            self._consume_unwanted_input()
+    def _skip_to_next_iteration_group(self):
+        space = self.space
+        while True:
+            if self.w_currkey is None:
+                pass
+            elif self.w_tgtkey is None:
+                break
+            else:
+                if not space.eq_w(self.w_tgtkey, self.w_currkey):
+                    break
 
-        if not self.started:
-            self.started = True
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                raise
+            w_newvalue = space.next(self.w_iterator)
+            if space.is_w(self.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
             else:
-                self.w_lookahead = w_obj
-                if self.w_fun is None:
-                    self.w_key = w_obj
-                else:
-                    self.w_key = self.space.call_function(self.w_fun, w_obj)
-                self.lookahead = True
+                w_newkey = space.call_function(self.w_keyfunc, w_newvalue)
 
-        self.new_group = False
-        w_iterator = W_GroupByIterator(self.space, self.index, self)
-        return self.space.newtuple([self.w_key, w_iterator])
-
-    def _consume_unwanted_input(self):
-        # Consume unwanted input until we reach the next group
-        try:
-            while True:
-                self.group_next(self.index)
-        except StopIteration:
-            pass
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
-    def group_next(self, group_index):
-        if group_index < self.index:
-            raise StopIteration
-        else:
-            if self.lookahead:
-                self.lookahead = False
-                return self.w_lookahead
-
-            try:
-                w_obj = self.space.next(self.w_iterable)
-            except OperationError as e:
-                if e.match(self.space, self.space.w_StopIteration):
-                    self.exhausted = True
-                    raise StopIteration
-                else:
-                    raise
-            else:
-                if self.w_fun is None:
-                    w_new_key = w_obj
-                else:
-                    w_new_key = self.space.call_function(self.w_fun, w_obj)
-                if self.space.eq_w(self.w_key, w_new_key):
-                    return w_obj
-                else:
-                    self.index += 1
-                    self.w_lookahead = w_obj
-                    self.w_key = w_new_key
-                    self.lookahead = True
-                    self.new_group = True #new group
-                    raise StopIteration
+            self.w_currkey = w_newkey
+            self.w_currvalue = w_newvalue
 
 def W_GroupBy___new__(space, w_subtype, w_iterable, w_key=None):
     r = space.allocate_instance(W_GroupBy, w_subtype)
@@ -1036,26 +988,34 @@
 
 
 class W_GroupByIterator(W_Root):
-    def __init__(self, space, index, groupby):
-        self.space = space
-        self.index = index
+    def __init__(self, groupby, w_tgtkey):
         self.groupby = groupby
-        self.exhausted = False
+        self.w_tgtkey = w_tgtkey
 
     def iter_w(self):
         return self
 
     def next_w(self):
-        if self.exhausted:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        groupby = self.groupby
+        space = groupby.space
+        if groupby.w_currvalue is None:
+            w_newvalue = space.next(groupby.w_iterator)
+            if space.is_w(groupby.w_keyfunc, space.w_None):
+                w_newkey = w_newvalue
+            else:
+                w_newkey = space.call_function(groupby.w_keyfunc, w_newvalue)
+            #assert groupby.w_currvalue is None
+            # ^^^ check disabled, see http://bugs.python.org/issue30347
+            groupby.w_currkey = w_newkey
+            groupby.w_currvalue = w_newvalue
 
-        try:
-            w_obj = self.groupby.group_next(self.index)
-        except StopIteration:
-            self.exhausted = True
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-        else:
-            return w_obj
+        assert groupby.w_currkey is not None
+        if not space.eq_w(self.w_tgtkey, groupby.w_currkey):
+            raise OperationError(space.w_StopIteration, space.w_None)
+        w_result = groupby.w_currvalue
+        groupby.w_currvalue = None
+        groupby.w_currkey = None
+        return w_result
 
 W_GroupByIterator.typedef = TypeDef(
         'itertools._groupby',
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -634,6 +634,17 @@
         it = itertools.groupby([0], 1)
         raises(TypeError, it.next)
 
+    def test_groupby_question_43905804(self):
+        # http://stackoverflow.com/questions/43905804/
+        import itertools
+
+        inputs = ((x > 5, x) for x in range(10))
+        (_, a), (_, b) = itertools.groupby(inputs, key=lambda x: x[0])
+        a = list(a)
+        b = list(b)
+        assert a == []
+        assert b == [(True, 9)]
+
     def test_iterables(self):
         import itertools
     
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -1,3 +1,4 @@
+from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import oefmt
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
@@ -21,7 +22,7 @@
 
 TimSort = make_timsort_class()
 class StrideSort(TimSort):
-    ''' 
+    '''
     argsort (return the indices to sort) a list of strides
     '''
     def __init__(self, rangelist, strides, order):
@@ -380,14 +381,14 @@
 
     def get_buffer(self, space, flags):
         errtype = space.w_ValueError # should be BufferError, numpy does this instead
-        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and 
+        if ((flags & space.BUF_C_CONTIGUOUS) == space.BUF_C_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_C_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not C-contiguous")
-        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and 
+        if ((flags & space.BUF_F_CONTIGUOUS) == space.BUF_F_CONTIGUOUS and
                 not self.flags & NPY.ARRAY_F_CONTIGUOUS):
            raise oefmt(errtype, "ndarray is not Fortran contiguous")
         if ((flags & space.BUF_ANY_CONTIGUOUS) == space.BUF_ANY_CONTIGUOUS and
-                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and 
+                not (self.flags & NPY.ARRAY_F_CONTIGUOUS and
                      self.flags & NPY.ARRAY_C_CONTIGUOUS)):
            raise oefmt(errtype, "ndarray is not contiguous")
         if ((flags & space.BUF_STRIDES) != space.BUF_STRIDES and
@@ -397,7 +398,7 @@
             not self.flags & NPY.ARRAY_WRITEABLE):
            raise oefmt(errtype, "buffer source array is read-only")
         readonly = not (flags & space.BUF_WRITABLE) == space.BUF_WRITABLE
-        return ArrayBuffer(self, readonly)
+        return ArrayView(self, readonly)
 
     def astype(self, space, dtype, order, copy=True):
         # copy the general pattern of the strides
@@ -527,7 +528,7 @@
         try:
             length = support.product_check(shape)
             self.size = ovfcheck(length * dtype.elsize)
-        except OverflowError: 
+        except OverflowError:
             raise oefmt(dtype.itemtype.space.w_ValueError, "array is too big.")
         if storage == lltype.nullptr(RAW_STORAGE):
             if dtype.num == NPY.OBJECT:
@@ -701,10 +702,8 @@
     def __del__(self):
         free_raw_storage(self.storage)
 
-
-class ArrayBuffer(Buffer):
+class ArrayData(Buffer):
     _immutable_ = True
-
     def __init__(self, impl, readonly):
         self.impl = impl
         self.readonly = readonly
@@ -725,6 +724,28 @@
         from rpython.rtyper.lltypesystem import rffi
         return rffi.ptradd(self.impl.storage, self.impl.start)
 
+
+class ArrayView(BufferView):
+    _immutable_ = True
+
+    def __init__(self, impl, readonly):
+        self.impl = impl
+        self.readonly = readonly
+        self.data = ArrayData(impl, readonly)
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def as_readbuf(self):
+        return ArrayData(self.impl, readonly=True)
+
+    def as_writebuf(self):
+        assert not self.readonly
+        return ArrayData(self.impl, readonly=False)
+
     def getformat(self):
         sb = StringBuilder()
         self.impl.dtype.getformat(sb)
@@ -742,4 +763,5 @@
     def getstrides(self):
         return self.impl.strides
 
-
+    def get_raw_address(self):
+        return self.data.get_raw_address()
diff --git a/pypy/module/micronumpy/ctors.py b/pypy/module/micronumpy/ctors.py
--- a/pypy/module/micronumpy/ctors.py
+++ b/pypy/module/micronumpy/ctors.py
@@ -91,7 +91,7 @@
             w_base = w_object
             if read_only:
                 w_base = None
-            return W_NDimArray.from_shape_and_storage(space, shape, w_data, 
+            return W_NDimArray.from_shape_and_storage(space, shape, w_data,
                                 dtype, w_base=w_base, strides=strides,
                                 start=offset), read_only
         if w_data is None:
@@ -104,11 +104,11 @@
         #print 'create view from shape',shape,'dtype',dtype,'data',data
         if strides is not None:
             raise oefmt(space.w_NotImplementedError,
-                   "__array_interface__ strides not fully supported yet") 
+                   "__array_interface__ strides not fully supported yet")
         arr = frombuffer(space, w_data, dtype, support.product(shape), offset)
         new_impl = arr.implementation.reshape(arr, shape)
         return W_NDimArray(new_impl), False
-        
+
     except OperationError as e:
         if e.match(space, space.w_AttributeError):
             return None, False
@@ -120,7 +120,7 @@
         return descr
     msg = "invalid PEP 3118 format string: '%s'" % c_format
     space.warn(space.newtext(msg), space.w_RuntimeWarning)
-    return None 
+    return None
 
 def _array_from_buffer_3118(space, w_object, dtype):
     try:
@@ -139,12 +139,12 @@
             raise oefmt(space.w_NotImplementedError,

From pypy.commits at gmail.com  Thu May 18 11:03:52 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Thu, 18 May 2017 08:03:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: a readme for the applevel tests in this
 dir
Message-ID: <591db7d8.9d451c0a.e7d3.5966@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91329:157a06a45851
Date: 2017-05-18 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/157a06a45851/

Log:	a readme for the applevel tests in this dir

diff --git a/pypy/module/test_lib_pypy/README.txt b/pypy/module/test_lib_pypy/README.txt
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/README.txt
@@ -0,0 +1,4 @@
+This directory contains app-level tests are supposed to be run *after*
+translation. So you run them by saying:
+
+pypy pytest.py <testfile.py>

From pypy.commits at gmail.com  Thu May 18 11:13:35 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:35 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a new test flag to
 disallow fast paths
Message-ID: <591dba1f.9d451c0a.e7d3.5c1d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91332:274be07c41af
Date: 2017-05-18 12:01 +0200
http://bitbucket.org/pypy/pypy/changeset/274be07c41af/

Log:	add a new test flag to disallow fast paths

diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -19,6 +19,7 @@
 
 USE_FASTPATH = True    # set to False by some tests
 ALLOW_SLOWPATH = True  # set to False by some tests
+ALLOW_FASTPATH = True  # set to False by some tests
 
 native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
@@ -39,6 +40,8 @@
             pos % size != 0):
             raise CannotWrite
         #
+        if not ALLOW_FASTPATH:
+            raise ValueError("fastpath not allowed :(")
         # typed_write() might raise CannotWrite
         fmtiter.wbuf.typed_write(TYPE, fmtiter.pos, value)
         fmtiter.advance(size)
@@ -208,6 +211,8 @@
             # *and* it is not supported.
             raise CannotRead
         #
+        if not ALLOW_FASTPATH:
+            raise ValueError("fastpath not allowed :(")
         # typed_read does not do any bound check, so we must call it only if
         # we are sure there are at least "size" bytes to read
         if fmtiter.can_advance(size):
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -44,14 +44,17 @@
 
     USE_FASTPATH = True
     ALLOW_SLOWPATH = True
+    ALLOW_FASTPATH = True
     
     def setup_method(self, meth):
         standardfmttable.USE_FASTPATH = self.USE_FASTPATH
         standardfmttable.ALLOW_SLOWPATH = self.ALLOW_SLOWPATH
+        standardfmttable.ALLOW_FASTPATH = self.ALLOW_FASTPATH
 
     def teardown_method(self, meth):
         standardfmttable.USE_FASTPATH = True
         standardfmttable.ALLOW_SLOWPATH = True
+        standardfmttable.ALLOW_FASTPATH = True
 
     def mypack(self, fmt, value):
         size = struct.calcsize(fmt)
@@ -94,6 +97,16 @@
         expected = struct.pack('i', 42)
         assert self.mypack('i', 42) == expected
 
+class TestAllowFastPath(PackSupport):
+    ALLOW_FASTPATH = False
+    bigendian = nativefmttable.native_is_bigendian
+    fmttable = standardfmttable.standard_fmttable
+
+    def test_fastpath_not_allowed(self):
+        # we are using a native endianess but ALLOW_FASTPATH is False, so
+        # the following MUST raise
+        pytest.raises(ValueError, "self.mypack('i', 42)")
+
 
 class BaseTestPack(PackSupport):
 

From pypy.commits at gmail.com  Thu May 18 11:13:31 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:31 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: ouch, this was plainly wrong,
 as it overwrote the correct teardown_method just above
Message-ID: <591dba1b.5987df0a.3cf9b.0b17@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91330:0265fea70804
Date: 2017-05-18 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/0265fea70804/

Log:	ouch, this was plainly wrong, as it overwrote the correct
	teardown_method just above

diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -53,12 +53,6 @@
         standardfmttable.USE_FASTPATH = True
         standardfmttable.ALLOW_SLOWPATH = True
 
-    def teardown_method(self, meth):
-        if not hasattr(self.fmttable, 'USE_FASTPATH'):
-            return
-        self.fmttable.USE_FASTPATH = self.orig_use_fastpath
-        self.fmttable.ALLOW_SLOWPATH = self.orig_allow_slowpath
-
     def mypack(self, fmt, value):
         size = struct.calcsize(fmt)
         fake_fmtiter = FakeFormatIter(self.bigendian, size, value)

From pypy.commits at gmail.com  Thu May 18 11:13:38 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:38 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: WIP: add a failing test which
 show why the current way to detect alignement is buggy
Message-ID: <591dba22.89321c0a.84945.027d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91334:6bed52a9d7c6
Date: 2017-05-18 16:26 +0200
http://bitbucket.org/pypy/pypy/changeset/6bed52a9d7c6/

Log:	WIP: add a failing test which show why the current way to detect
	alignement is buggy

diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -2,6 +2,7 @@
 from rpython.rlib.rarithmetic import r_ulonglong
 from rpython.rlib.rstruct import standardfmttable, nativefmttable
 from rpython.rlib.rstruct.error import StructOverflowError
+from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.mutbuffer import MutableStringBuffer
 import struct
 
@@ -215,3 +216,17 @@
         assert fake_fmtiter.pos == wbuf.getlength()
         got = wbuf.finish()
         assert got == expected
+
+    def test_subbuffer(self):
+        # to force a non-aligned 'i'
+        expected = struct.pack('=BBi', 0xAB, 0xCD, 0x1234)
+        size = len(expected)
+        #
+        wbuf = MutableStringBuffer(size)
+        wsubbuf = SubBuffer(wbuf, 2, size-4)
+        wbuf.setitem(0, chr(0xAB))
+        wbuf.setitem(1, chr(0xCD))
+        fake_fmtiter = self.mypack_into('i', wsubbuf, 0x1234)
+        assert fake_fmtiter.pos == wbuf.getlength()
+        got = wbuf.finish()
+        assert got == expected

From pypy.commits at gmail.com  Thu May 18 11:13:33 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:33 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: write tests to check that
 USE_FASTPATH and ALLOW_SLOWPATH actually do what they are supposed to do
Message-ID: <591dba1d.86aedf0a.255a5.c5f8@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91331:ce82d5756ddd
Date: 2017-05-18 11:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ce82d5756ddd/

Log:	write tests to check that USE_FASTPATH and ALLOW_SLOWPATH actually
	do what they are supposed to do

diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -31,7 +31,7 @@
         raise AttributeError(name)
 
 
-class BaseTestPack(object):
+class PackSupport(object):
     """
     These test tests only the various pack_* functions, individually.  There
     is no RPython interface to them, as for now they are used only to
@@ -71,6 +71,32 @@
         got = self.mypack(fmt, value)
         assert got == expected
 
+
+class TestAllowSlowpath(PackSupport):
+    ALLOW_SLOWPATH = False
+    bigendian = not nativefmttable.native_is_bigendian
+    fmttable = standardfmttable.standard_fmttable
+
+    def test_slowpath_not_allowed(self):
+        # we are using a non-native endianess and ALLOW_SLOWPATH is False, so
+        # the following MUST raise
+        pytest.raises(ValueError, "self.mypack('i', 42)")
+
+
+class TestUseFastpath(PackSupport):
+    ALLOW_SLOWPATH = False
+    bigendian = nativefmttable.native_is_bigendian
+    fmttable = standardfmttable.standard_fmttable
+
+    def test_fastpath_taken(self):
+        # we are using native endianess and slowpath is not allowed, so the
+        # following MUST succeed
+        expected = struct.pack('i', 42)
+        assert self.mypack('i', 42) == expected
+
+
+class BaseTestPack(PackSupport):
+
     def test_pack_int(self):
         self.check('b', 42)
         self.check('B', 242)
@@ -153,3 +179,4 @@
     bigendian = nativefmttable.native_is_bigendian
     fmt_prefix = '@'
     fmttable = nativefmttable.native_fmttable
+

From pypy.commits at gmail.com  Thu May 18 11:13:40 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:40 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: delegate the alignment check
 to Buffer.typed_{read, write},
 to take in account also the SubBuffer's offset. Moreover,
 reuse the logic in rawstorage to determine whether it is fine to do an
 unaligned access, depending on the CPU
Message-ID: <591dba24.d588df0a.c4cca.1cca@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91335:7d8f1bbfc0b1
Date: 2017-05-18 17:12 +0200
http://bitbucket.org/pypy/pypy/changeset/7d8f1bbfc0b1/

Log:	delegate the alignment check to Buffer.typed_{read,write}, to take
	in account also the SubBuffer's offset. Moreover, reuse the logic in
	rawstorage to determine whether it is fine to do an unaligned
	access, depending on the CPU

diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -13,6 +13,21 @@
                               ll_for_resizable_list)
 from rpython.rlib.signature import signature
 from rpython.rlib import types
+from rpython.rlib import rawstorage
+
+ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
+
+ at specialize.ll()
+def is_alignment_correct(TYPE, index):
+    if ALLOW_UNALIGNED_ACCESS:
+        return True
+    try:
+        rawstorage._check_alignment(TYPE, index)
+    except rawstorage.AlignmentError:
+        return False
+    else:
+        return True
+
 
 class CannotRead(Exception):
     """
@@ -117,6 +132,8 @@
         """
         Read the value of type TP starting at byte_offset. No bounds checks
         """
+        if not is_alignment_correct(TP, byte_offset):
+            raise CannotRead
         ptr = self.get_raw_address()
         return llop.raw_load(TP, ptr, byte_offset)
 
@@ -125,7 +142,7 @@
         """
         Write the value of type TP at byte_offset. No bounds checks
         """
-        if self.readonly:
+        if self.readonly or not is_alignment_correct(TP, byte_offset):
             raise CannotWrite
         ptr = self.get_raw_address()
         value = lltype.cast_primitive(TP, value)
@@ -158,6 +175,8 @@
 
         @specialize.ll_and_arg(1)
         def typed_read(self, TP, byte_offset):
+            if not is_alignment_correct(TP, byte_offset):
+                raise CannotRead
             lldata = self._get_gc_data()
             byte_offset += self._get_gc_data_extra_offset()
             return llop.gc_load_indexed(TP, lldata, byte_offset,
@@ -165,7 +184,7 @@
 
         @specialize.ll_and_arg(1)
         def typed_write(self, TP, byte_offset, value):
-            if self.readonly:
+            if self.readonly or not is_alignment_correct(TP, byte_offset):
                 raise CannotWrite
             lldata = self._get_gc_data()
             byte_offset += self._get_gc_data_extra_offset()
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -33,17 +33,17 @@
     @specialize.argtype(0)
     def do_pack_fastpath(fmtiter, value):
         size = rffi.sizeof(TYPE)
-        pos = fmtiter.pos
         if (not USE_FASTPATH or
             fmtiter.bigendian != native_is_bigendian or
-            not native_is_ieee754 or
-            pos % size != 0):
+            not native_is_ieee754):
             raise CannotWrite
         #
-        if not ALLOW_FASTPATH:
-            raise ValueError("fastpath not allowed :(")
         # typed_write() might raise CannotWrite
         fmtiter.wbuf.typed_write(TYPE, fmtiter.pos, value)
+        if not ALLOW_FASTPATH:
+            # if we are here it means that typed_write did not raise, and thus
+            # the fast path was actually taken
+            raise ValueError("fastpath not allowed :(")
         fmtiter.advance(size)
     #
     @specialize.argtype(0)
@@ -203,12 +203,7 @@
     def do_unpack_fastpath(fmtiter):
         size = rffi.sizeof(TYPE)
         buf, pos = fmtiter.get_buffer_and_pos()
-        if pos % size != 0 or not USE_FASTPATH:
-            # XXX: maybe we are too conservative here? On most architectures,
-            # it is possible to read the data even if pos is not
-            # aligned. Also, probably it should responsibility of
-            # buf.typed_read to raise CannotRead in case it is not aligned
-            # *and* it is not supported.
+        if not USE_FASTPATH:
             raise CannotRead
         #
         if not ALLOW_FASTPATH:
diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -2,8 +2,10 @@
 from rpython.rlib.rarithmetic import r_ulonglong
 from rpython.rlib.rstruct import standardfmttable, nativefmttable
 from rpython.rlib.rstruct.error import StructOverflowError
+from rpython.rlib import buffer
 from rpython.rlib.buffer import SubBuffer
 from rpython.rlib.mutbuffer import MutableStringBuffer
+from rpython.rlib import rawstorage
 import struct
 
 class FakeFormatIter(object):
@@ -40,16 +42,19 @@
     USE_FASTPATH = True
     ALLOW_SLOWPATH = True
     ALLOW_FASTPATH = True
+    ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
     
     def setup_method(self, meth):
         standardfmttable.USE_FASTPATH = self.USE_FASTPATH
         standardfmttable.ALLOW_SLOWPATH = self.ALLOW_SLOWPATH
         standardfmttable.ALLOW_FASTPATH = self.ALLOW_FASTPATH
+        buffer.ALLOW_UNALIGNED_ACCESS = self.ALLOW_UNALIGNED_ACCESS
 
     def teardown_method(self, meth):
         standardfmttable.USE_FASTPATH = True
         standardfmttable.ALLOW_SLOWPATH = True
         standardfmttable.ALLOW_FASTPATH = True
+        buffer.ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
 
     def mypack(self, fmt, value):
         size = struct.calcsize(fmt)
@@ -202,6 +207,7 @@
 
 class TestUnaligned(PackSupport):
     ALLOW_FASTPATH = False
+    ALLOW_UNALIGNED_ACCESS = False
     bigendian = nativefmttable.native_is_bigendian
     fmttable = nativefmttable.native_fmttable
 
@@ -227,6 +233,6 @@
         wbuf.setitem(0, chr(0xAB))
         wbuf.setitem(1, chr(0xCD))
         fake_fmtiter = self.mypack_into('i', wsubbuf, 0x1234)
-        assert fake_fmtiter.pos == wbuf.getlength()
+        assert fake_fmtiter.pos == wbuf.getlength()-2 # -2 since it's a SubBuffer
         got = wbuf.finish()
         assert got == expected

From pypy.commits at gmail.com  Thu May 18 11:13:37 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:13:37 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a passing test to check
 that we don't take the fast path for unaligned access
Message-ID: <591dba21.c486df0a.b241a.9492@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91333:23b951353efe
Date: 2017-05-18 16:19 +0200
http://bitbucket.org/pypy/pypy/changeset/23b951353efe/

Log:	add a passing test to check that we don't take the fast path for
	unaligned access

diff --git a/rpython/rlib/rstruct/test/test_pack.py b/rpython/rlib/rstruct/test/test_pack.py
--- a/rpython/rlib/rstruct/test/test_pack.py
+++ b/rpython/rlib/rstruct/test/test_pack.py
@@ -7,21 +7,15 @@
 
 class FakeFormatIter(object):
 
-    def __init__(self, bigendian, size, value):
-        from rpython.rlib.rstring import StringBuilder
+    def __init__(self, bigendian, wbuf, value):
         self.value = value
         self.bigendian = bigendian
-        self.wbuf = MutableStringBuffer(size)
+        self.wbuf = wbuf
         self.pos = 0
 
     def advance(self, count):
         self.pos += count
 
-    def finish(self):
-        # check that we called advance() the right number of times
-        assert self.pos == self.wbuf.getlength()
-        return self.wbuf.finish()
-
     def _accept_arg(self):
         return self.value
 
@@ -58,16 +52,27 @@
 
     def mypack(self, fmt, value):
         size = struct.calcsize(fmt)
-        fake_fmtiter = FakeFormatIter(self.bigendian, size, value)
+        wbuf = MutableStringBuffer(size)
+        fake_fmtiter = self.mypack_into(fmt, wbuf, value)
+        # check that we called advance() the right number of times
+        assert fake_fmtiter.pos == wbuf.getlength()
+        return wbuf.finish()
+
+    def mypack_into(self, fmt, wbuf, value, advance=None):
+        fake_fmtiter = FakeFormatIter(self.bigendian, wbuf, value)
+        if advance:
+            fake_fmtiter.advance(advance)
         attrs = self.fmttable[fmt]
         pack = attrs['pack']
         pack(fake_fmtiter)
-        return fake_fmtiter.finish()
+        return fake_fmtiter
 
     def mypack_fn(self, func, size, arg, value):
-        fmtiter = FakeFormatIter(self.bigendian, size, value)
-        func(fmtiter, arg)
-        return fmtiter.finish()
+        wbuf = MutableStringBuffer(size)
+        fake_fmtiter = FakeFormatIter(self.bigendian, wbuf, value)
+        func(fake_fmtiter, arg)
+        assert fake_fmtiter.pos == wbuf.getlength()
+        return wbuf.finish()
 
     def check(self, fmt, value):
         expected = struct.pack(self.fmt_prefix+fmt, value)
@@ -193,3 +198,20 @@
     fmt_prefix = '@'
     fmttable = nativefmttable.native_fmttable
 
+
+class TestUnaligned(PackSupport):
+    ALLOW_FASTPATH = False
+    bigendian = nativefmttable.native_is_bigendian
+    fmttable = nativefmttable.native_fmttable
+
+    def test_unaligned(self):
+        # to force a non-aligned 'i'
+        expected = struct.pack('=BBi', 0xAB, 0xCD, 0x1234)
+        #
+        wbuf = MutableStringBuffer(len(expected))
+        wbuf.setitem(0, chr(0xAB))
+        wbuf.setitem(1, chr(0xCD))
+        fake_fmtiter = self.mypack_into('i', wbuf, 0x1234, advance=2)
+        assert fake_fmtiter.pos == wbuf.getlength()
+        got = wbuf.finish()
+        assert got == expected

From pypy.commits at gmail.com  Thu May 18 11:27:07 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 18 May 2017 08:27:07 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: hg merge default
Message-ID: <591dbd4b.a193df0a.266b6.ded4@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91336:5dbafd0b241b
Date: 2017-05-18 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/5dbafd0b241b/

Log:	hg merge default

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -42,8 +42,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -677,7 +677,7 @@
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
-                      PyClassObject'''.split():
+                      PyClassObject PyBaseExceptionObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
 
     if w_dict is None:
         return 0
-
+    if not space.isinstance_w(w_dict, space.w_dict):
+        return 0 
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         Py_IncRef(space, py_dict.c__tmpkeys)
     else:
+        if not py_dict.c__tmpkeys:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
         w_keys = from_ref(space, py_dict.c__tmpkeys)
     ppos[0] += 1
     if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,7 +14,7 @@
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
@@ -264,7 +264,7 @@
     check_num_args(space, w_args, 1)
     args_w = space.fixedview(w_args)
     index = space.int_w(space.index(args_w[0]))
-    null = lltype.nullptr(PyObject.TO)
+    null = rffi.cast(PyObject, 0)
     res = generic_cpy_call(space, func_target, w_self, index, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
@@ -293,7 +293,8 @@
     func_target = rffi.cast(objobjargproc, func)
     check_num_args(space, w_args, 1)
     w_key, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, None)
+    null = rffi.cast(PyObject, 0)
+    res = generic_cpy_call(space, func_target, w_self, w_key, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.w_None
@@ -611,6 +612,8 @@
             handled = True
 
     for tp_name, attr in [('tp_hash', '__hash__'),
+                          ('tp_as_sequence.c_sq_length', '__len__'),
+                          ('tp_as_mapping.c_mp_length', '__len__'),
                          ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -636,7 +639,8 @@
                           ('tp_as_number.c_nb_xor', '__xor__'),
                           ('tp_as_number.c_nb_or', '__or__'),
                           ('tp_as_sequence.c_sq_concat', '__add__'),
-                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+                          ('tp_as_mapping.c_mp_subscript', '__getitem__'),
                           ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -650,7 +654,7 @@
             handled = True
 
     # binary-with-Py_ssize_t-type
-    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -679,7 +683,48 @@
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
             handled = True
+    # ternary-with-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
 
+            @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, w_arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, w_arg1)
+                return 0
+            handled = True
+    # ternary-Py_size_t-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
+
+            @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, space.newint(arg1))
+                return 0
+            handled = True
     if handled:
         pass
     elif name == 'tp_setattro':
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -255,4 +255,60 @@
             ])
         d = module.get_type_dict(1)
         assert d['real'].__get__(1, 1) == 1
-
+    def test_advanced(self):
+        module = self.import_extension('foo', [
+            ("dict_len", "METH_O",
+            '''
+                int ret = args->ob_type->tp_as_mapping->mp_length(args);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_setitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 3 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1),
+                        PyTuple_GetItem(args, 2));
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_delitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 2 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1), NULL);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_next", "METH_VARARGS",
+            '''
+                PyObject *key, *value;
+                PyObject *arg = NULL;
+                Py_ssize_t pos = 0;
+                int ret = 0;
+                if ((PyArg_ParseTuple(args, "|O", &arg))) {
+                    if (arg && PyDict_Check(arg)) {
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                        /* test no crash if pos is not reset to 0*/
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                    }
+                }
+                return PyLong_FromLong(ret);
+            '''),
+            ])
+        d = {'a': 1, 'b':2}
+        assert module.dict_len(d) == 2
+        assert module.dict_setitem(d, 'a', 'c') == 0
+        assert d['a'] == 'c'
+        assert module.dict_delitem(d, 'a') == 0
+        r = module.dict_next({'a': 1, 'b': 2})
+        assert r == 2
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -520,7 +520,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+ at cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
diff --git a/pypy/module/test_lib_pypy/README.txt b/pypy/module/test_lib_pypy/README.txt
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/README.txt
@@ -0,0 +1,4 @@
+This directory contains app-level tests are supposed to be run *after*
+translation. So you run them by saying:
+
+pypy pytest.py <testfile.py>
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -239,7 +239,7 @@
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
         includes.append('sys/sysmacros.h')
-    if sys.platform.startswith('freebsd'):
+    if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'):
         includes.append('sys/ttycom.h')
     libraries = ['util']
 eci = ExternalCompilationInfo(
diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c
--- a/rpython/rlib/rvmprof/src/shared/machine.c
+++ b/rpython/rlib/rvmprof/src/shared/machine.c
@@ -4,6 +4,7 @@
 #include <stdio.h>
 
 #ifdef VMPROF_UNIX
+#include <string.h>
 #include <unistd.h>
 #include <fcntl.h>
 #endif

From pypy.commits at gmail.com  Thu May 18 17:05:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 18 May 2017 14:05:03 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: OS/X => OS X
Message-ID: <591e0c7f.10d91c0a.63d9b.36a1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r885:f8278c922a6e
Date: 2017-05-18 23:04 +0200
http://bitbucket.org/pypy/pypy.org/changeset/f8278c922a6e/

Log:	OS/X => OS X

diff --git a/compat.html b/compat.html
--- a/compat.html
+++ b/compat.html
@@ -140,7 +140,7 @@
 the program finishes running in the meantime).  See <a class="reference external" href="http://pypy.readthedocs.org/en/latest/cpython_differences.html#differences-related-to-garbage-collection-strategies">more details
 here</a>.</p>
 <p>Note that PyPy returns unused memory to the operating system if there
-is a madvise() system call (at least Linux, OS/X, BSD) or on Windows.  It is
+is a madvise() system call (at least Linux, OS X, BSD) or on Windows.  It is
 important to realize that you may not see this in <tt class="docutils literal">top</tt>.  The unused
 pages are marked with <tt class="docutils literal">MADV_FREE</tt>, which tells the system &ldquo;if you
 need more memory at some point, grab this page&rdquo;.  As long as memory is
diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -75,7 +75,7 @@
 as stable as the release, but they contain numerous bugfixes and
 performance improvements.</p>
 <p>We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
-Linux, Mac OS/X and Windows:</p>
+Linux, Mac OS X and Windows:</p>
 <ul class="simple">
 <li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.7.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.7.1.html">what's new in PyPy2.7?</a>)</li>
 <li>the Python3.5 compatible beta quality release &#8212; <strong>PyPy3.5 v5.7.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.7.1.html">what's new in PyPy3.5?</a>).</li>
@@ -125,7 +125,7 @@
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-osx64.tar.bz2">Mac OS/X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
diff --git a/source/compat.txt b/source/compat.txt
--- a/source/compat.txt
+++ b/source/compat.txt
@@ -131,7 +131,7 @@
 here`_.
 
 Note that PyPy returns unused memory to the operating system if there
-is a madvise() system call (at least Linux, OS/X, BSD) or on Windows.  It is
+is a madvise() system call (at least Linux, OS X, BSD) or on Windows.  It is
 important to realize that you may not see this in ``top``.  The unused
 pages are marked with ``MADV_FREE``, which tells the system "if you
 need more memory at some point, grab this page".  As long as memory is
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -13,7 +13,7 @@
   performance improvements.
 
 We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
-Linux, Mac OS/X and Windows:
+Linux, Mac OS X and Windows:
 
 * the Python2.7 compatible release &#8212; **PyPy2.7 v5.7.1** &#8212; (`what's new in PyPy2.7?`_)
 
@@ -87,7 +87,7 @@
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below)
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)
 * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)`__ (see ``[1]`` below)
-* `Mac OS/X binary (64bit)`__
+* `Mac OS X binary (64bit)`__
 * FreeBSD x86 and x86_64: see FreshPorts_
 * `Windows binary (32bit)`__ (you might need the VS 2008 runtime library
   installer `vcredist_x86.exe`_.)
diff --git a/source/tmdonate2.txt b/source/tmdonate2.txt
--- a/source/tmdonate2.txt
+++ b/source/tmdonate2.txt
@@ -278,14 +278,14 @@
 heavily on Linux- and clang-only features.  We believe it is a suitable
 restriction: a lot of multi- and many-core servers commonly available
 are nowadays x86-64 machines running Linux.  Nevertheless, non-Linux
-solutions appear to be possible as well.  OS/X (and likely the various
+solutions appear to be possible as well.  OS X (and likely the various
 BSDs) seems to handle ``mmap()`` better than Linux does, and can remap
 individual pages of an existing mapping to various pages without hitting
 a limit of 65536 like Linux.  Windows might also have a solution, although we
 didn't measure yet; but first we would need a 64-bit Windows PyPy, which has
 not seen much active support.
 
-We will likely explore the OS/X path (as well as the Windows path if Win64
+We will likely explore the OS X path (as well as the Windows path if Win64
 support grows in PyPy), but this is not part of this current
 donation proposal.
 
diff --git a/tmdonate2.html b/tmdonate2.html
--- a/tmdonate2.html
+++ b/tmdonate2.html
@@ -283,13 +283,13 @@
 heavily on Linux- and clang-only features.  We believe it is a suitable
 restriction: a lot of multi- and many-core servers commonly available
 are nowadays x86-64 machines running Linux.  Nevertheless, non-Linux
-solutions appear to be possible as well.  OS/X (and likely the various
+solutions appear to be possible as well.  OS X (and likely the various
 BSDs) seems to handle <tt class="docutils literal">mmap()</tt> better than Linux does, and can remap
 individual pages of an existing mapping to various pages without hitting
 a limit of 65536 like Linux.  Windows might also have a solution, although we
 didn't measure yet; but first we would need a 64-bit Windows PyPy, which has
 not seen much active support.</p>
-<p>We will likely explore the OS/X path (as well as the Windows path if Win64
+<p>We will likely explore the OS X path (as well as the Windows path if Win64
 support grows in PyPy), but this is not part of this current
 donation proposal.</p>
 <p>It might be possible to adapt the work done on x86-64 to the 64-bit

From pypy.commits at gmail.com  Thu May 18 22:31:22 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 May 2017 19:31:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Work around issue #2560
Message-ID: <591e58fa.028b1c0a.f2c6e.6e63@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91337:dc9dca0f58fb
Date: 2017-05-19 03:03 +0100
http://bitbucket.org/pypy/pypy/changeset/dc9dca0f58fb/

Log:	Work around issue #2560

diff --git a/pypy/module/cpyext/test/test_version.py b/pypy/module/cpyext/test/test_version.py
--- a/pypy/module/cpyext/test/test_version.py
+++ b/pypy/module/cpyext/test/test_version.py
@@ -37,7 +37,8 @@
         Py_RETURN_NONE;
         """
         module = self.import_module(name='foo', init=init)
-        assert module.py_version == '%d.%d.%d' % sys.version_info[:3]
+        expected = '%d.%d.%d' % sys.version_info[:3]
+        assert module.py_version == expected
         assert module.py_major_version == sys.version_info.major
         assert module.py_minor_version == sys.version_info.minor
         assert module.py_micro_version == sys.version_info.micro

From pypy.commits at gmail.com  Thu May 18 22:36:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 18 May 2017 19:36:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix PY_VERSION inconsistency
Message-ID: <591e5a17.c486df0a.b8723.a972@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91338:e3545a45e827
Date: 2017-05-19 03:33 +0100
http://bitbucket.org/pypy/pypy/changeset/e3545a45e827/

Log:	Fix PY_VERSION inconsistency

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -26,7 +26,7 @@
 #define PY_RELEASE_SERIAL	0
 
 /* Version as a string */
-#define PY_VERSION		"3.5.2"
+#define PY_VERSION		"3.5.3"
 
 /* PyPy version as a string */
 #define PYPY_VERSION "5.8.0-alpha0"

From pypy.commits at gmail.com  Fri May 19 05:52:26 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 19 May 2017 02:52:26 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: rpython fix for 32 bit
Message-ID: <591ec05a.eaacdf0a.4580c.37a7@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91339:acd0aa896e2d
Date: 2017-05-19 11:51 +0200
http://bitbucket.org/pypy/pypy/changeset/acd0aa896e2d/

Log:	rpython fix for 32 bit

diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -6,7 +6,7 @@
 
 from rpython.rlib import jit, longlong2float
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_singlefloat, widen
+from rpython.rlib.rarithmetic import r_singlefloat, widen, intmask
 from rpython.rlib.rstruct import standardfmttable as std
 from rpython.rlib.rstruct.standardfmttable import native_is_bigendian
 from rpython.rlib.rstruct.error import StructError
@@ -44,6 +44,7 @@
     # slow path
     value = longlong2float.singlefloat2uint(floatval)
     value = widen(value)
+    value = intmask(value)
     pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 4, fmtiter.bigendian)
     fmtiter.advance(4)
 

From pypy.commits at gmail.com  Fri May 19 06:09:51 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 19 May 2017 03:09:51 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix the expected JIT ops after
 the recent W_BytearrayObject changes
Message-ID: <591ec46f.8f081c0a.a37cc.c761@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91340:1a6ceca7adce
Date: 2017-05-19 12:07 +0200
http://bitbucket.org/pypy/pypy/changeset/1a6ceca7adce/

Log:	fix the expected JIT ops after the recent W_BytearrayObject changes

diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -142,9 +142,12 @@
         log = self.run(main, [1000])
         assert log.result == main(1000)
         loop, = log.loops_by_filename(self.filepath)
+        # the offset of gc_load_indexed_i used to be the constant 0. However,
+        # now it is 'i46' because we need to add 0 to
+        # W_BytearrayObject._offset
         assert loop.match_by_id('unpack', """
             guard_not_invalidated(descr=...)
-            i70 = gc_load_indexed_i(p48, 0, 1, _, -2)
+            i70 = gc_load_indexed_i(p48, i46, 1, _, -2)
         """)
 
     def test_pack_into_raw_buffer(self):
@@ -191,5 +194,6 @@
             i77 = int_le(i62, 32767)
             guard_true(i77, descr=...)
             p78 = getfield_gc_r(p68, descr=<FieldP list.items \d+>)
-            gc_store_indexed(p78, 4, i62, 1, _, 2, descr=<ArrayS 2>)
+            i81 = int_add(4, i70)
+            gc_store_indexed(p78, i81, i62, 1, _, 2, descr=<ArrayS 2>)
         """)

From pypy.commits at gmail.com  Fri May 19 08:53:04 2017
From: pypy.commits at gmail.com (Dodan)
Date: Fri, 19 May 2017 05:53:04 -0700 (PDT)
Subject: [pypy-commit] pypy pypy_ctypes_nosegfault_nofastpath: fixed ctypes
 segfault by removing fastpath. Test in test_segfault.py
Message-ID: <591eeab0.12891c0a.7d1f5.384b@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: pypy_ctypes_nosegfault_nofastpath
Changeset: r91341:96435a57d046
Date: 2017-05-18 17:27 +0300
http://bitbucket.org/pypy/pypy/changeset/96435a57d046/

Log:	fixed ctypes segfault by removing fastpath. Test in test_segfault.py

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,9 +26,9 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
-WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+WIN64 = sys.platform == 'win32' and sys.maxint == 2 ** 63 - 1
 
 
 def get_com_error(errcode, riid, pIunk):
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -94,14 +97,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -126,27 +124,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -156,7 +153,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -168,15 +165,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -188,7 +188,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -201,7 +201,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -222,10 +222,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -259,7 +257,7 @@
         if (sys.platform == 'win32' and isinstance(argument, (int, long))
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -281,6 +279,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -317,7 +316,7 @@
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
-                print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
+                print >> sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
             if self._restype_ is not None:
                 return res
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/pypy/module/test_lib_pypy/ctypes_tests/README
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/README
@@ -0,0 +1,8 @@
+-------Ctypes tests------
+
+Unlike the other tests in the PyPy sources, these tests are assumed to run after the translation is complete.
+Therefore, using the resulting binary, you can then call, for example:
+
+/path/to/your_modified_pypy_binary pytest.py pypy/module/test_lib_pypy/ctypes_tests/test_libc.py
+
+This also applies to any other test in ctypes_tests.
\ No newline at end of file
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py b/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from ctypes import CDLL, POINTER, pointer, c_byte, c_int, c_char_p, CFUNCTYPE, c_void_p, c_size_t
-import sys
-import py
-from support import BaseCTypesTestChecker
-
-class MyCDLL(CDLL):
-    def __getattr__(self, attr):
-        fn = self[attr] # this way it's not cached as an attribute
-        fn._slowpath_allowed = False
-        return fn
-
-def setup_module(mod):
-    import conftest
-    _ctypes_test = str(conftest.sofile)
-    mod.dll = MyCDLL(_ctypes_test)  # slowpath not allowed
-    mod.dll2 = CDLL(_ctypes_test)   # slowpath allowed
-
-
-class TestFastpath(BaseCTypesTestChecker):
-
-    def test_fastpath_forbidden(self):
-        def myfunc():
-            pass
-        #
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        #
-        # so far, it's still using the slowpath
-        assert not tf_b._is_fastpath
-        tf_b.callable = myfunc
-        tf_b.argtypes = (c_byte,)
-        # errcheck prevented the fastpath to kick in
-        assert not tf_b._is_fastpath
-        #
-        del tf_b.callable
-        tf_b.argtypes = (c_byte,) # try to re-enable the fastpath
-        assert tf_b._is_fastpath
-        #
-        assert not tf_b._slowpath_allowed
-        py.test.raises(AssertionError, "tf_b.callable = myfunc")
-        py.test.raises(AssertionError, "tf_b('aaa')") # force a TypeError
-
-    def test_simple_args(self):
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        assert tf_b(-126) == -42
-
-    def test_from_cfunctype(self):
-        from _ctypes import _memmove_addr
-        functype = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)
-        my_memmove = functype(_memmove_addr)
-        assert my_memmove._is_fastpath
-
-    def test_undeclared_restype(self):
-        # make sure we get a fresh function
-        try:
-            del dll.tf_i
-        except AttributeError:
-            pass
-        tf_i = dll.tf_i
-        assert not tf_i._is_fastpath
-        tf_i.argtypes = (c_int,)
-        assert tf_i._is_fastpath
-        assert tf_i(12) == 4
-
-    def test_pointer_args(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-        v = c_int(42)
-        result = f(pointer(v))
-        assert type(result) == POINTER(c_int)
-        assert result.contents.value == 42
-
-    def test_simple_pointer_args(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        mystr = c_char_p("abcd")
-        result = f(mystr, ord("b"))
-        assert result == "bcd"
-
-    def test_strings(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        result = f("abcd", ord("b"))
-        assert result == "bcd"
-
-    def test_errcheck(self):
-        def errcheck(result, func, args):
-            return 'hello'
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.errcheck = errcheck
-        assert tf_b(-126) == 'hello'
-
-    def test_array_to_ptr(self):
-        ARRAY = c_int * 8
-        func = dll._testfunc_ai8
-        func.restype = POINTER(c_int)
-        func.argtypes = [ARRAY]
-        array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8)
-        ptr = func(array)
-        assert ptr[0] == 1
-        assert ptr[7] == 8
-
-
-class TestFallbackToSlowpath(BaseCTypesTestChecker):
-
-    def test_argtypes_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_char_p,)  # this is intentionally wrong
-        tf_b.argtypes = None # kill the fast path
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -42
-
-    def test_callable_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.callable = lambda x: x+1
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -125
-        tf_b.callable = None
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py b/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
@@ -0,0 +1,16 @@
+# The fast path has been removed and so were the tests
+# They have been replaced with a test that shows that passing the wrong argument to a function that takes a string parameter no longer segfaults.
+# This is why the assertion of an exception below. If it fails then it segfaults.
+
+import ctypes
+
+class TestFastpath():
+
+    def test_fastpath_forbidden(self):
+        libc = ctypes.cdll.LoadLibrary("libc.so.6")
+        libc.strlen.argtypes = [ctypes.c_char_p]
+        libc.strlen.restype = ctypes.c_size_t
+        try:
+            libc.strlen(False)
+        except Exception as e:
+                assert isinstance(e,Exception)

From pypy.commits at gmail.com  Fri May 19 08:53:10 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 19 May 2017 05:53:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 Dodan/pypy_ctypes/pypy_ctypes_nosegfault_nofastpath (pull request #547)
Message-ID: <591eeab6.b885df0a.8c62b.ffb5@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91344:8b4ebb4d87c8
Date: 2017-05-19 12:52 +0000
http://bitbucket.org/pypy/pypy/changeset/8b4ebb4d87c8/

Log:	Merged in Dodan/pypy_ctypes/pypy_ctypes_nosegfault_nofastpath (pull
	request #547)

	fixed ctypes segfault by removing fastpath.

	Approved-by: Carl Friedrich Bolz <cfbolz at gmx.de> Approved-by:
	Antonio Cuni <anto.cuni+2 at gmail.com>

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,9 +26,9 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
-WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+WIN64 = sys.platform == 'win32' and sys.maxint == 2 ** 63 - 1
 
 
 def get_com_error(errcode, riid, pIunk):
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -94,14 +97,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -126,27 +124,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -156,7 +153,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -168,15 +165,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -188,7 +188,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -201,7 +201,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -222,10 +222,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -259,7 +257,7 @@
         if (sys.platform == 'win32' and isinstance(argument, (int, long))
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -281,6 +279,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -317,7 +316,7 @@
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
-                print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
+                print >> sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
             if self._restype_ is not None:
                 return res
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/pypy/module/test_lib_pypy/ctypes_tests/README
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/README
@@ -0,0 +1,8 @@
+-------Ctypes tests------
+
+Unlike the other tests in the PyPy sources, these tests are assumed to run after the translation is complete.
+Therefore, using the resulting binary, you can then call, for example:
+
+/path/to/your_modified_pypy_binary pytest.py pypy/module/test_lib_pypy/ctypes_tests/test_libc.py
+
+This also applies to any other test in ctypes_tests.
\ No newline at end of file
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py b/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from ctypes import CDLL, POINTER, pointer, c_byte, c_int, c_char_p, CFUNCTYPE, c_void_p, c_size_t
-import sys
-import py
-from support import BaseCTypesTestChecker
-
-class MyCDLL(CDLL):
-    def __getattr__(self, attr):
-        fn = self[attr] # this way it's not cached as an attribute
-        fn._slowpath_allowed = False
-        return fn
-
-def setup_module(mod):
-    import conftest
-    _ctypes_test = str(conftest.sofile)
-    mod.dll = MyCDLL(_ctypes_test)  # slowpath not allowed
-    mod.dll2 = CDLL(_ctypes_test)   # slowpath allowed
-
-
-class TestFastpath(BaseCTypesTestChecker):
-
-    def test_fastpath_forbidden(self):
-        def myfunc():
-            pass
-        #
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        #
-        # so far, it's still using the slowpath
-        assert not tf_b._is_fastpath
-        tf_b.callable = myfunc
-        tf_b.argtypes = (c_byte,)
-        # errcheck prevented the fastpath to kick in
-        assert not tf_b._is_fastpath
-        #
-        del tf_b.callable
-        tf_b.argtypes = (c_byte,) # try to re-enable the fastpath
-        assert tf_b._is_fastpath
-        #
-        assert not tf_b._slowpath_allowed
-        py.test.raises(AssertionError, "tf_b.callable = myfunc")
-        py.test.raises(AssertionError, "tf_b('aaa')") # force a TypeError
-
-    def test_simple_args(self):
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        assert tf_b(-126) == -42
-
-    def test_from_cfunctype(self):
-        from _ctypes import _memmove_addr
-        functype = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)
-        my_memmove = functype(_memmove_addr)
-        assert my_memmove._is_fastpath
-
-    def test_undeclared_restype(self):
-        # make sure we get a fresh function
-        try:
-            del dll.tf_i
-        except AttributeError:
-            pass
-        tf_i = dll.tf_i
-        assert not tf_i._is_fastpath
-        tf_i.argtypes = (c_int,)
-        assert tf_i._is_fastpath
-        assert tf_i(12) == 4
-
-    def test_pointer_args(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-        v = c_int(42)
-        result = f(pointer(v))
-        assert type(result) == POINTER(c_int)
-        assert result.contents.value == 42
-
-    def test_simple_pointer_args(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        mystr = c_char_p("abcd")
-        result = f(mystr, ord("b"))
-        assert result == "bcd"
-
-    def test_strings(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        result = f("abcd", ord("b"))
-        assert result == "bcd"
-
-    def test_errcheck(self):
-        def errcheck(result, func, args):
-            return 'hello'
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.errcheck = errcheck
-        assert tf_b(-126) == 'hello'
-
-    def test_array_to_ptr(self):
-        ARRAY = c_int * 8
-        func = dll._testfunc_ai8
-        func.restype = POINTER(c_int)
-        func.argtypes = [ARRAY]
-        array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8)
-        ptr = func(array)
-        assert ptr[0] == 1
-        assert ptr[7] == 8
-
-
-class TestFallbackToSlowpath(BaseCTypesTestChecker):
-
-    def test_argtypes_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_char_p,)  # this is intentionally wrong
-        tf_b.argtypes = None # kill the fast path
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -42
-
-    def test_callable_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.callable = lambda x: x+1
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -125
-        tf_b.callable = None
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -478,7 +478,7 @@
         raises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
 
     def test_argument_conversion_and_checks(self):
-        py.test.skip("XXX currently broken on PyPy, sorry")
+        #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
         strlen = dll.my_strchr
         strlen.argtypes = [c_char_p, c_int]
         strlen.restype = c_char_p

From pypy.commits at gmail.com  Fri May 19 08:53:06 2017
From: pypy.commits at gmail.com (Dodan)
Date: Fri, 19 May 2017 05:53:06 -0700 (PDT)
Subject: [pypy-commit] pypy pypy_ctypes_nosegfault_nofastpath:
 Test_segfault.py now throws ctypes.ArgumentError
Message-ID: <591eeab2.83821c0a.2260c.4374@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: pypy_ctypes_nosegfault_nofastpath
Changeset: r91342:a0c92b42599a
Date: 2017-05-19 10:43 +0300
http://bitbucket.org/pypy/pypy/changeset/a0c92b42599a/

Log:	Test_segfault.py now throws ctypes.ArgumentError

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py b/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
@@ -12,5 +12,5 @@
         libc.strlen.restype = ctypes.c_size_t
         try:
             libc.strlen(False)
-        except Exception as e:
-                assert isinstance(e,Exception)
+        except ctypes.ArgumentError as e:
+                assert isinstance(e,ctypes.ArgumentError)

From pypy.commits at gmail.com  Fri May 19 08:53:08 2017
From: pypy.commits at gmail.com (Dodan)
Date: Fri, 19 May 2017 05:53:08 -0700 (PDT)
Subject: [pypy-commit] pypy pypy_ctypes_nosegfault_nofastpath: Removed
 test_segfault.py. The test was in test_functions.py in
 test_argument_conversion_and_checks
Message-ID: <591eeab4.e3b2df0a.de960.307c@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: pypy_ctypes_nosegfault_nofastpath
Changeset: r91343:5823134aebb6
Date: 2017-05-19 15:40 +0300
http://bitbucket.org/pypy/pypy/changeset/5823134aebb6/

Log:	Removed test_segfault.py. The test was in test_functions.py in
	test_argument_conversion_and_checks

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -478,7 +478,7 @@
         raises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
 
     def test_argument_conversion_and_checks(self):
-        py.test.skip("XXX currently broken on PyPy, sorry")
+        #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
         strlen = dll.my_strchr
         strlen.argtypes = [c_char_p, c_int]
         strlen.restype = c_char_p
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py b/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_segfault.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# The fast path has been removed and so were the tests
-# They have been replaced with a test that shows that passing the wrong argument to a function that takes a string parameter no longer segfaults.
-# This is why the assertion of an exception below. If it fails then it segfaults.
-
-import ctypes
-
-class TestFastpath():
-
-    def test_fastpath_forbidden(self):
-        libc = ctypes.cdll.LoadLibrary("libc.so.6")
-        libc.strlen.argtypes = [ctypes.c_char_p]
-        libc.strlen.restype = ctypes.c_size_t
-        try:
-            libc.strlen(False)
-        except ctypes.ArgumentError as e:
-                assert isinstance(e,ctypes.ArgumentError)

From pypy.commits at gmail.com  Fri May 19 10:01:17 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 19 May 2017 07:01:17 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: bah,
 the two tests shared the same cache,
 which means that they worked if invoked separately but broke if you tested
 the whole file
Message-ID: <591efaad.4699df0a.e57c5.41f3@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91345:ca5fc3f48609
Date: 2017-05-19 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/ca5fc3f48609/

Log:	bah, the two tests shared the same cache, which means that they
	worked if invoked separately but broke if you tested the whole file

diff --git a/rpython/translator/c/test/test_llop.py b/rpython/translator/c/test/test_llop.py
--- a/rpython/translator/c/test/test_llop.py
+++ b/rpython/translator/c/test/test_llop.py
@@ -5,10 +5,11 @@
 
 
 class TestLLOp(BaseLLOpTest):
-    cache = {}
+    cache_load = {}
+    cache_store = {}
 
     def gc_load_from_string(self, TYPE, buf, offset):
-        if TYPE not in self.cache:
+        if TYPE not in self.cache_load:
             assert isinstance(TYPE, lltype.Primitive)
             if TYPE in (lltype.Float, lltype.SingleFloat):
                 TARGET_TYPE = lltype.Float
@@ -20,14 +21,14 @@
                 return lltype.cast_primitive(TARGET_TYPE, x)
 
             fn = compile(llf, [str, int])
-            self.cache[TYPE] = fn
+            self.cache_load[TYPE] = fn
         #
-        fn = self.cache[TYPE]
+        fn = self.cache_load[TYPE]
         x = fn(buf, offset)
         return lltype.cast_primitive(TYPE, x)
 
     def newlist_and_gc_store(self, TYPE, value, expected):
-        if TYPE not in self.cache:
+        if TYPE not in self.cache_store:
             assert isinstance(TYPE, lltype.Primitive)
             if TYPE in (lltype.Float, lltype.SingleFloat):
                 argtype = float
@@ -40,8 +41,8 @@
                 return ''.join(lst)
 
             fn = compile(llf, [argtype])
-            self.cache[TYPE] = fn
+            self.cache_store[TYPE] = fn
         #
-        fn = self.cache[TYPE]
+        fn = self.cache_store[TYPE]
         got = fn(value)
         assert got == expected

From pypy.commits at gmail.com  Sat May 20 05:55:23 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sat, 20 May 2017 02:55:23 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix writeanalyze.py to take
 into account llop.gc_store_indexed
Message-ID: <5920128b.59b0df0a.e5a49.5209@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91346:85d3ab6fe80b
Date: 2017-05-20 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/85d3ab6fe80b/

Log:	fix writeanalyze.py to take into account llop.gc_store_indexed

diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -269,6 +269,34 @@
         assert struct == "struct"
         assert name.endswith("foobar")
 
+    def test_gc_store_indexed(self):
+        from rpython.rtyper.lltypesystem import lltype, llmemory
+        from rpython.rtyper.lltypesystem.lloperation import llop
+        from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+                                      ll_for_resizable_list)
+
+        def write_item(lst, byte_offset, value):
+            ll_data = ll_for_resizable_list(lst)
+            ll_items = ll_data.items
+            LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
+            base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
+            scale_factor = llmemory.sizeof(lltype.Char)
+            llop.gc_store_indexed(lltype.Void, ll_items, byte_offset, value,
+                                  scale_factor, base_ofs)
+
+        def f(x):
+            lst = resizable_list_supporting_raw_ptr(['A', 'B', 'C', 'D'])
+            write_item(lst, 0, 'E')
+        t, wa = self.translate(f, [int])
+        fgraph = graphof(t, f)
+        result = wa.analyze(fgraph.startblock.operations[-1])
+        #
+        assert len(result) == 1
+        array, A = list(result)[0]
+        assert array == "array"
+        assert A.TO.OF == lltype.Char
+
+
 
 class TestLLtypeReadWriteAnalyze(BaseTest):
     Analyzer = ReadWriteAnalyzer
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -61,6 +61,9 @@
             if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]):
                 name = self._getinteriorname(op)
                 return self._interiorfield_result(op.args[0].concretetype, name)
+        elif op.opname == "gc_store_indexed":
+            if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]):
+                return self._array_result(op.args[0].concretetype)
         return empty_set
 
     def _array_result(self, TYPE):

From pypy.commits at gmail.com  Sat May 20 10:36:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 20 May 2017 07:36:55 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-obj-stealing: Tweaks and simplifications
Message-ID: <59205487.5788df0a.610a9.b979@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cpyext-obj-stealing
Changeset: r91347:fb0a61fd753d
Date: 2017-05-20 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/fb0a61fd753d/

Log:	Tweaks and simplifications

	A few "pointless" operations have been added which I removed again;
	I'm unsure if the point was only to make the test pass (I needed to
	tweak it) or if there was more.

diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,10 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import (decref, incref, PyObject, make_ref, 
-            w_obj_has_pyobj)
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -21,11 +21,18 @@
     setting all items to a real object with PyList_SetItem().
     """
     w_list = space.newlist([None] * len)
-    w_list.convert_to_cpy_strategy(space)
+    #w_list.convert_to_cpy_strategy(space)
     return w_list
 
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    assert isinstance(w_list, W_ListObject)
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
 @cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
-def PyList_SET_ITEM(space, w_list, index, w_item):
+def PyList_SET_ITEM(space, w_list, index, py_item):
     """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
@@ -33,22 +40,15 @@
     discard a reference to any item that it being replaced; any reference in
     list at position i will be leaked.
     """
-    from pypy.module.cpyext.sequence import CPyListStrategy
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    assert isinstance(w_list, W_ListObject)
-    assert w_list.strategy is cpy_strategy, "list strategy not CPyListStrategy"
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    w_old = w_list.getitem(index)
-    incref(space, w_old) # since setitem calls decref, maintain cpython semantics here
-    w_list.setitem(index, w_item)
-    decref(space, w_item)
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
@@ -58,12 +58,19 @@
     if index < 0 or index >= w_list.length():
         decref(space, w_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.convert_to_cpy_strategy(space)
-    w_list.setitem(index, w_item)
-    decref(space, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
@@ -73,18 +80,15 @@
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    w_list.convert_to_cpy_strategy(space)
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
 def PyList_Append(space, w_list, w_item):
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
-    #pyobj = make_ref(space, w_item)
     w_list.append(w_item)
-    w_list.convert_to_cpy_strategy(space)  # calls incref when switching strategy
     return 0
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
@@ -92,7 +96,6 @@
     """Insert the item item into list list in front of index index.  Return
     0 if successful; return -1 and set an exception if unsuccessful.
     Analogous to list.insert(index, item)."""
-    make_ref(space, w_item)
     space.call_method(space.w_list, "insert", w_list, space.newint(index), w_item)
     return 0
 
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -276,14 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
-        assert pyobj.c_ob_refcnt > 0
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
-        if not pyobj:
-            keepalive_until_here(obj)
-            return pyobj
-        assert pyobj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
-    pyobj.c_ob_refcnt += 1
+        at_least = rawrefcount.REFCNT_FROM_PYPY
+    if pyobj:
+        assert pyobj.c_ob_refcnt >= at_least
+        pyobj.c_ob_refcnt += 1
+        keepalive_until_here(obj)
     return pyobj
 
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,11 +38,13 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
-    
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
+
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
@@ -204,7 +206,7 @@
     def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
                 /* test that the refcount differences for functions
                  * are correct. diff1 - expected refcount diff for i1,
@@ -230,33 +232,38 @@
 
                 PyObject* tmp, *o = PyList_New(0);
                 char errbuffer[1024];
-                PyObject* i1 = PyBytes_FromString("random string 1");
-                PyObject* i2 = PyBytes_FromString("random string 2");
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
                 Py_ssize_t old_count1, new_count1;
                 Py_ssize_t old_count2, new_count2;
                 Py_ssize_t diff;
                 int ret;
 
-                Py_INCREF(i2); // since it is used in macros
-
-                old_count1 = Py_REFCNT(i1); // 1
-                old_count2 = Py_REFCNT(i2); // 1
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
                 ret = PyList_Append(o, i1);
                 if (ret != 0) 
                     return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
                 CHECKCOUNT(1, 0, "PyList_Append");
 
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
+
                 PyList_SET_ITEM(o, 0, i2);
                 CHECKCOUNT(0, 0, "PyList_SET_ITEM");
 
                 tmp = PyList_GET_ITEM(o, 0);
-                // XXX should tmp == i2?
-                if ((Py_REFCNT(tmp) != Py_REFCNT(i2))) 
+                if (tmp != i2)
                 {
-                    sprintf(errbuffer, "GETITEM return (%ld) and i2 (%ld)refcounts"
-                            " unequal", (long)Py_REFCNT(tmp), (long)Py_REFCNT(i2));
-                    PyErr_SetString(PyExc_AssertionError, errbuffer); 
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
                     return NULL;
                 }
                 CHECKCOUNT(0, 0, "PyList_GET_ITEM");
@@ -274,8 +281,6 @@
                     CHECKCOUNT(-1, 0, "Py_DECREF(o)");
                 }
                 #endif
-                Py_DECREF(i1); // append incref'd.
-                Py_DECREF(i2); 
                 return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -237,10 +237,6 @@
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self

From pypy.commits at gmail.com  Sat May 20 15:41:57 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 May 2017 12:41:57 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: test,
 fix for unhashable c-api objects
Message-ID: <59209c05.1b8bdf0a.69246.23b2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91348:552622605578
Date: 2017-05-20 22:30 +0300
http://bitbucket.org/pypy/pypy/changeset/552622605578/

Log:	test, fix for unhashable c-api objects

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -272,6 +272,7 @@
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCWrapperObject),
     __repr__ = interp2app(W_PyCWrapperObject.descr_method_repr),
     # XXX missing: __getattribute__
+    __hash__ = None
     )
 W_PyCWrapperObject.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -159,3 +159,11 @@
         fd = BytesIO()
         # only test that it works
         fd.write(a)
+
+    def test_hash(self):
+        module = self.import_module(name='array')
+        a = module.array('c')
+        exc = raises(TypeError, hash, a)
+        assert 'unhashable' in str(exc.value)
+        exc = raises(TypeError, hash, a.__mul__)
+        assert 'unhashable' in str(exc.value)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -338,7 +338,6 @@
                 setattr(struct, slot_names[1], slot_func_helper)
 
 def add_operators(space, dict_w, pto):
-    # XXX support PyObject_HashNotImplemented
     for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers:
         if method_name in dict_w:
             continue
@@ -365,6 +364,9 @@
             rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
+    if not pto.c_tp_hash:
+        dict_w['__hash__'] = space.w_None
+    print 'pto', rffi.charp2str(pto.c_tp_name), 'c_tp_hash', pto.c_tp_hash, dict_w['__hash__']
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
@@ -943,6 +945,7 @@
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
     w_dict = w_obj.getdict(space)
+
     # pass in the w_obj to convert any values that are
     # unbound GetSetProperty into bound PyGetSetDescrObject
     pto.c_tp_dict = make_ref(space, w_dict, w_obj)

From pypy.commits at gmail.com  Sat May 20 15:41:59 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 May 2017 12:41:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cpyext-obj-stealing which
 improves PyListObject refcounting compatibility
Message-ID: <59209c07.91d81c0a.c63a0.6e07@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91349:158d97f6e1eb
Date: 2017-05-20 22:31 +0300
http://bitbucket.org/pypy/pypy/changeset/158d97f6e1eb/

Log:	merge cpyext-obj-stealing which improves PyListObject refcounting
	compatibility

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1557,7 +1557,6 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,9 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -19,59 +20,68 @@
     PySequence_SetItem()  or expose the object to Python code before
     setting all items to a real object with PyList_SetItem().
     """
-    return space.newlist([None] * len)
+    w_list = space.newlist([None] * len)
+    #w_list.convert_to_cpy_strategy(space)
+    return w_list
 
- at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
-             result_borrowed=True)
-def PyList_SET_ITEM(space, w_list, index, w_item):
-    """Macro form of PyList_SetItem() without error checking. This is normally
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    assert isinstance(w_list, W_ListObject)
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
+def PyList_SET_ITEM(space, w_list, index, py_item):
+    """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
-    This function "steals" a reference to item, and, unlike PyList_SetItem(),
-    does not discard a reference to any item that it being replaced; any
-    reference in list at position i will be leaked.
+    "steals" a reference to item, and, unlike PyList_SetItem(), does not
+    discard a reference to any item that it being replaced; any reference in
+    list at position i will be leaked.
     """
-    assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    # Deliberately leak, so that it can be safely decref'd.
-    make_ref(space, w_list.getitem(index))
-    Py_DecRef(space, w_item)
-    w_list.setitem(index, w_item)
-    return w_item
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
-    Py_DecRef(space, w_item)
     if not isinstance(w_list, W_ListObject):
+        decref(space, w_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
+        decref(space, w_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.setitem(index, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
-    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    if w_list.strategy is not cpy_strategy:
-        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -30,8 +30,7 @@
         return subtype_dealloc.api_func
 
     def allocate(self, space, w_type, itemcount=0):
-        # similar to PyType_GenericAlloc?
-        # except that it's not related to any pypy object.
+        # typically called from PyType_GenericAlloc via typedescr.allocate
         # this returns a PyObject with ob_refcnt == 1.
 
         pytype = as_pyobj(space, w_type)
@@ -250,6 +249,8 @@
     w_obj = rawrefcount.to_obj(W_Root, pyobj)
     return w_obj is not None and w_obj is not w_marker_deallocating
 
+def w_obj_has_pyobj(w_obj):
+    return bool(rawrefcount.from_obj(PyObject, w_obj))
 
 def is_pyobj(x):
     if x is None or isinstance(x, W_Root):
@@ -275,13 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
+        at_least = rawrefcount.REFCNT_FROM_PYPY
     if pyobj:
-        assert pyobj.c_ob_refcnt > 0
+        assert pyobj.c_ob_refcnt >= at_least
         pyobj.c_ob_refcnt += 1
-        if not is_pyobj(obj):
-            keepalive_until_here(obj)
+        keepalive_until_here(obj)
     return pyobj
 
 
@@ -315,9 +317,14 @@
         obj = rffi.cast(PyObject, obj)
         if obj:
             assert obj.c_ob_refcnt > 0
+            assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            #else:
+            #    w_obj = rawrefcount.to_obj(W_Root, ref)
+            #    if w_obj is not None:
+            #        assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
     else:
         get_w_obj_and_decref(space, obj)
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -1872,6 +1872,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -1887,6 +1888,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,11 +38,13 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
-    
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
+
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
@@ -152,29 +154,35 @@
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
-            ("test_macro_invocations", "METH_NOARGS",
+            ("test_macro_invocations", "METH_O",
              """
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
+             PyList_SET_ITEM(o, 0, args);
+             PyList_SET_ITEM(l, 1, args);
 
-             Py_INCREF(o);
-             PyList_SET_ITEM(o, 0, o);
-             Py_INCREF(o);
-             PyList_SET_ITEM(l, 1, o);
+             if(PyList_GET_ITEM(o, 0) != PyList_GET_ITEM(l, 1))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_ITEM error");
+                return NULL;
+             }
 
-             PyList_GET_ITEM(o, 0);
-             PyList_GET_ITEM(l, 1);
-
-             PyList_GET_SIZE(o);
-             PyList_GET_SIZE(l);
+             if(PyList_GET_SIZE(o) != PyList_GET_SIZE(l))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_SIZE error");
+                return NULL;
+             }
 
              return o;
              """
             )
         ])
-        x = module.test_macro_invocations()
-        assert x[0] is x[1] is x
+        s = 'abcdef'
+        x = module.test_macro_invocations(s)
+        assert x[0] is x[1] is s
 
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
@@ -183,39 +191,96 @@
                 PyObject* o, *o2, *o3;
                 o = PyList_New(1);
 
-                o2 = PyInt_FromLong(0);
+                o2 = PyBytes_FromString("test_get_item0");
+                Py_INCREF(o2);
                 PyList_SET_ITEM(o, 0, o2);
-                o2 = NULL;
 
                 o3 = PyList_GET_ITEM(o, 0);
                 Py_INCREF(o3);
-                Py_CLEAR(o);
+                Py_DECREF(o);
+                Py_DECREF(o2);
                 return o3;
              """)])
-        assert module.test_get_item() == 0
+        assert module.test_get_item() == b'test_get_item0'
 
-    def test_set_item_macro(self):
+    def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff_after_setitem", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
-                PyObject* o = PyList_New(0);
-                PyObject* o2 = PyList_New(0);
-                Py_ssize_t refcount, new_refcount;
+                /* test that the refcount differences for functions
+                 * are correct. diff1 - expected refcount diff for i1,
+                 *              diff2 - expected refcount diff for i2
+                 */
+                #define CHECKCOUNT(diff1, diff2, action) \
+                    new_count1 = Py_REFCNT(i1); \
+                    new_count2 = Py_REFCNT(i2); \
+                    diff = new_count1 - old_count1; \
+                    if (diff != diff1) {\
+                        sprintf(errbuffer, action \
+                            " i1 expected diff of %ld got %ld", (long)diff1, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    diff = new_count2 - old_count2; \
+                    if (diff != diff2) {\
+                        sprintf(errbuffer, action \
+                            " i2 expected diff of %ld got %ld", (long)diff2, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    old_count1 = new_count1; \
+                    old_count2 = new_count2;
 
-                PyList_Append(o, o2);  // does not steal o2
+                PyObject* tmp, *o = PyList_New(0);
+                char errbuffer[1024];
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
+                Py_ssize_t old_count1, new_count1;
+                Py_ssize_t old_count2, new_count2;
+                Py_ssize_t diff;
+                int ret;
 
-                refcount = Py_REFCNT(o2);
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
-                // Steal a reference to o2, but leak the old reference to o2.
-                // The net result should be no change in refcount.
-                PyList_SET_ITEM(o, 0, o2);
+                ret = PyList_Append(o, i1);
+                if (ret != 0) 
+                    return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(1, 0, "PyList_Append");
 
-                new_refcount = Py_REFCNT(o2);
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
 
-                Py_CLEAR(o);
-                Py_DECREF(o2); // append incref'd.
-                // Py_CLEAR(o2);  // naive implementation would fail here.
-                return PyLong_FromSsize_t(new_refcount - refcount);
+                PyList_SET_ITEM(o, 0, i2);
+                CHECKCOUNT(0, 0, "PyList_SET_ITEM");
+
+                tmp = PyList_GET_ITEM(o, 0);
+                if (tmp != i2)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(0, 0, "PyList_GET_ITEM");
+
+                PyList_SetItem(o, 0, i1);
+                CHECKCOUNT(0, -1, "PyList_Set_Item");
+
+                PyList_GetItem(o, 0);
+                CHECKCOUNT(0, 0, "PyList_Get_Item");
+
+                Py_DECREF(o); 
+                #ifndef PYPY_VERSION
+                {
+                    // PyPy deletes only at teardown
+                    CHECKCOUNT(-1, 0, "Py_DECREF(o)");
+                }
+                #endif
+                return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff_after_setitem() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -24,7 +24,7 @@
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
+    PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, Py_DecRef, as_pyobj)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -230,15 +230,13 @@
         return list(items)
 
     def switch_to_object_strategy(self):
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        if self.strategy is object_strategy:
+            return
         list_w = self.getitems()
-        object_strategy = self.space.fromcache(ObjectListStrategy)
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self

From pypy.commits at gmail.com  Sat May 20 15:42:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 May 2017 12:42:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge cpyext-recursionlimit which
 implements Py_EnterRecursiveCall and friends
Message-ID: <59209c09.84ae1c0a.1e926.908a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91350:17fddf817604
Date: 2017-05-20 22:34 +0300
http://bitbucket.org/pypy/pypy/changeset/17fddf817604/

Log:	merge cpyext-recursionlimit which implements Py_EnterRecursiveCall
	and friends

diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +229,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, widen(limit))
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -490,29 +490,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyFileObject], lltype.Void)
 def PyFile_IncUseCount(space, p):
     """Increments the PyFileObject's internal use count to indicate
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -331,3 +331,30 @@
                 def nested_flags():
                     return module.get_flags()""" in ns
         assert ns['nested_flags']() == (1, 0x2000)  # CO_FUTURE_DIVISION
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  

From pypy.commits at gmail.com  Sat May 20 15:42:03 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 May 2017 12:42:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branches
Message-ID: <59209c0b.01a31c0a.7ec00.7e8e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91351:dba4bf058212
Date: 2017-05-20 22:40 +0300
http://bitbucket.org/pypy/pypy/changeset/dba4bf058212/

Log:	document merged branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,20 @@
 
 Internal refactoring of buffers and memoryviews. Memoryviews will now be
 accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+

From pypy.commits at gmail.com  Sat May 20 16:10:42 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 20 May 2017 13:10:42 -0700 (PDT)
Subject: [pypy-commit] pypy strbufobject: A branch to play with
 W_StringBufferObject (recently deleted on default)
Message-ID: <5920a2c2.4b171c0a.ec76.0d71@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: strbufobject
Changeset: r91353:fb38f51ca654
Date: 2017-05-20 21:08 +0100
http://bitbucket.org/pypy/pypy/changeset/fb38f51ca654/

Log:	A branch to play with W_StringBufferObject (recently deleted on
	default)


From pypy.commits at gmail.com  Sat May 20 16:10:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 20 May 2017 13:10:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Remove unused translation option
 'withstrbuf' and supporting code
Message-ID: <5920a2c0.0798df0a.ee57e.86ed@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91352:c74e33eeef0d
Date: 2017-05-20 20:58 +0100
http://bitbucket.org/pypy/pypy/changeset/c74e33eeef0d/

Log:	Remove unused translation option 'withstrbuf' and supporting code

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -220,9 +220,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -611,55 +611,31 @@
         return mod_format(space, w_values, self, do_unicode=False)
 
     def descr_eq(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -677,18 +653,6 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -81,9 +81,6 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
-        if self.config.objspace.std.withstrbuf:
-            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
-
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
@@ -285,7 +282,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/strbufobject.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import SimpleView, StringBuffer
-from pypy.interpreter.error import OperationError
-from rpython.rlib.rstring import StringBuilder
-
-
-class W_StringBufferObject(W_AbstractBytesObject):
-    w_str = None
-
-    def __init__(self, builder):
-        self.builder = builder             # StringBuilder
-        self.length = builder.getlength()
-
-    def force(self):
-        if self.w_str is None:
-            s = self.builder.build()
-            if self.length < len(s):
-                s = s[:self.length]
-            self.w_str = W_BytesObject(s)
-            return s
-        else:
-            return self.w_str._value
-
-    def __repr__(self):
-        """ representation for debugging purposes """
-        return "%s(%r[:%d])" % (
-            self.__class__.__name__, self.builder, self.length)
-
-    def unwrap(self, space):
-        return self.force()
-
-    def str_w(self, space):
-        return self.force()
-
-    def buffer_w(self, space, flags):
-        return SimpleView(StringBuffer(self.force()))
-
-    def descr_len(self, space):
-        return space.newint(self.length)
-
-    def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
-        else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
-
-    def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
-        return self
-
-
-delegation_dict = {}
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
-
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
-
-W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -57,13 +57,6 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
-    def test_withstrbuf_fastpath_isinstance(self):
-        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
-
-        space = gettestobjspace(withstrbuf=True)
-        cls = space._get_interplevel_cls(space.w_bytes)
-        assert cls is W_AbstractBytesObject
-
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import py
-
-from pypy.objspace.std.test import test_bytesobject
-
-class AppTestStringObject(test_bytesobject.AppTestBytesObject):
-    spaceconfig = {"objspace.std.withstrbuf": True}
-
-    def test_basic(self):
-        import __pypy__
-        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
-        # away on AST level
-        s = "Hello, ".__add__("World!")
-        assert type(s) is str
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-
-    def test_add_twice(self):
-        x = "a".__add__("b")
-        y = x + "c"
-        c = x + "d"
-        assert y == "abc"
-        assert c == "abd"
-
-    def test_add(self):
-        import __pypy__
-        all = ""
-        for i in range(20):
-            all += str(i)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
-        assert all == "012345678910111213141516171819"
-
-    def test_hash(self):
-        import __pypy__
-        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
-        t = 'a' * 101
-        s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-        assert hash(s) == hash(t)
-
-    def test_len(self):
-        s = "a".__add__("b")
-        r = "c".__add__("d")
-        t = s + r
-        assert len(s) == 2
-        assert len(r) == 2
-        assert len(t) == 4
-
-    def test_buffer(self):
-        s = b'a'.__add__(b'b')
-        assert buffer(s) == buffer(b'ab')
-        assert memoryview(s) == b'ab'
-
-    def test_add_strbuf(self):
-        # make three strbuf objects
-        s = 'a'.__add__('b')
-        t = 'x'.__add__('c')
-        u = 'y'.__add__('d')
-
-        # add two different strbufs to the same string
-        v = s + t
-        w = s + u
-
-        # check that insanity hasn't resulted.
-        assert v == "abxc"
-        assert w == "abyd"
-
-    def test_more_adding_fun(self):
-        s = 'a'.__add__('b') # s is a strbuf now
-        t = s + 'c'
-        u = s + 'd'
-        v = s + 'e'
-        assert v == 'abe'
-        assert u == 'abd'
-        assert t == 'abc'
-
-    def test_buh_even_more(self):
-        a = 'a'.__add__('b')
-        b = a + 'c'
-        c = '0'.__add__('1')
-        x = c + a
-        assert x == '01ab'
-
-    def test_add_non_string(self):
-        a = 'a'
-        a += 'b'
-        raises(TypeError, "a += 5")

From pypy.commits at gmail.com  Sat May 20 16:10:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 20 May 2017 13:10:45 -0700 (PDT)
Subject: [pypy-commit] pypy strbufobject: backout c74e33eeef0d: undelete
 W_StringBufferObject and withstrbuf config option
Message-ID: <5920a2c5.4297df0a.83a1.33cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: strbufobject
Changeset: r91354:8c65231e844b
Date: 2017-05-20 21:09 +0100
http://bitbucket.org/pypy/pypy/changeset/8c65231e844b/

Log:	backout c74e33eeef0d: undelete W_StringBufferObject and withstrbuf
	config option

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -220,6 +220,9 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
+        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
+                   default=False),
+
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -611,31 +611,55 @@
         return mod_format(space, w_values, self, do_unicode=False)
 
     def descr_eq(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            if isinstance(w_other, W_StringBufferObject):
+                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -653,6 +677,18 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
+        if space.config.objspace.std.withstrbuf:
+            from pypy.objspace.std.strbufobject import W_StringBufferObject
+            try:
+                other = self._op_val(space, w_other)
+            except OperationError as e:
+                if e.match(space, space.w_TypeError):
+                    return space.w_NotImplemented
+                raise
+            builder = StringBuilder()
+            builder.append(self._value)
+            builder.append(other)
+            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_BytesObject
+from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -81,6 +81,9 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
+        if self.config.objspace.std.withstrbuf:
+            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
+
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
@@ -282,7 +285,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/strbufobject.py
@@ -0,0 +1,96 @@
+import inspect
+
+import py
+
+from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.interpreter.gateway import interp2app, unwrap_spec
+from pypy.interpreter.buffer import SimpleView, StringBuffer
+from pypy.interpreter.error import OperationError
+from rpython.rlib.rstring import StringBuilder
+
+
+class W_StringBufferObject(W_AbstractBytesObject):
+    w_str = None
+
+    def __init__(self, builder):
+        self.builder = builder             # StringBuilder
+        self.length = builder.getlength()
+
+    def force(self):
+        if self.w_str is None:
+            s = self.builder.build()
+            if self.length < len(s):
+                s = s[:self.length]
+            self.w_str = W_BytesObject(s)
+            return s
+        else:
+            return self.w_str._value
+
+    def __repr__(self):
+        """ representation for debugging purposes """
+        return "%s(%r[:%d])" % (
+            self.__class__.__name__, self.builder, self.length)
+
+    def unwrap(self, space):
+        return self.force()
+
+    def str_w(self, space):
+        return self.force()
+
+    def buffer_w(self, space, flags):
+        return SimpleView(StringBuffer(self.force()))
+
+    def descr_len(self, space):
+        return space.newint(self.length)
+
+    def descr_add(self, space, w_other):
+        try:
+            other = W_BytesObject._op_val(space, w_other)
+        except OperationError as e:
+            if e.match(space, space.w_TypeError):
+                return space.w_NotImplemented
+            raise
+        if self.builder.getlength() != self.length:
+            builder = StringBuilder()
+            builder.append(self.force())
+        else:
+            builder = self.builder
+        builder.append(other)
+        return W_StringBufferObject(builder)
+
+    def descr_str(self, space):
+        # you cannot get subclasses of W_StringBufferObject here
+        assert type(self) is W_StringBufferObject
+        return self
+
+
+delegation_dict = {}
+for key, value in W_BytesObject.typedef.rawdict.iteritems():
+    if not isinstance(value, interp2app):
+        continue
+    if key in ('__len__', '__add__', '__str__'):
+        continue
+
+    func = value._code._bltin
+    args = inspect.getargs(func.func_code)
+    if args.varargs or args.keywords:
+        raise TypeError("Varargs and keywords not supported in unwrap_spec")
+    argspec = ', '.join([arg for arg in args.args[1:]])
+    func_code = py.code.Source("""
+    def f(self, %(args)s):
+        self.force()
+        return self.w_str.%(func_name)s(%(args)s)
+    """ % {'args': argspec, 'func_name': func.func_name})
+    d = {}
+    exec func_code.compile() in d
+    f = d['f']
+    f.func_defaults = func.func_defaults
+    f.__module__ = func.__module__
+    # necessary for unique identifiers for pickling
+    f.func_name = func.func_name
+    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
+    if unwrap_spec_ is not None:
+        f = unwrap_spec(**unwrap_spec_)(f)
+    setattr(W_StringBufferObject, func.func_name, f)
+
+W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -57,6 +57,13 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
+    def test_withstrbuf_fastpath_isinstance(self):
+        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
+
+        space = gettestobjspace(withstrbuf=True)
+        cls = space._get_interplevel_cls(space.w_bytes)
+        assert cls is W_AbstractBytesObject
+
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/objspace/std/test/test_strbufobject.py
@@ -0,0 +1,85 @@
+import py
+
+from pypy.objspace.std.test import test_bytesobject
+
+class AppTestStringObject(test_bytesobject.AppTestBytesObject):
+    spaceconfig = {"objspace.std.withstrbuf": True}
+
+    def test_basic(self):
+        import __pypy__
+        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
+        # away on AST level
+        s = "Hello, ".__add__("World!")
+        assert type(s) is str
+        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
+
+    def test_add_twice(self):
+        x = "a".__add__("b")
+        y = x + "c"
+        c = x + "d"
+        assert y == "abc"
+        assert c == "abd"
+
+    def test_add(self):
+        import __pypy__
+        all = ""
+        for i in range(20):
+            all += str(i)
+        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
+        assert all == "012345678910111213141516171819"
+
+    def test_hash(self):
+        import __pypy__
+        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
+        t = 'a' * 101
+        s = join(t)
+        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
+        assert hash(s) == hash(t)
+
+    def test_len(self):
+        s = "a".__add__("b")
+        r = "c".__add__("d")
+        t = s + r
+        assert len(s) == 2
+        assert len(r) == 2
+        assert len(t) == 4
+
+    def test_buffer(self):
+        s = b'a'.__add__(b'b')
+        assert buffer(s) == buffer(b'ab')
+        assert memoryview(s) == b'ab'
+
+    def test_add_strbuf(self):
+        # make three strbuf objects
+        s = 'a'.__add__('b')
+        t = 'x'.__add__('c')
+        u = 'y'.__add__('d')
+
+        # add two different strbufs to the same string
+        v = s + t
+        w = s + u
+
+        # check that insanity hasn't resulted.
+        assert v == "abxc"
+        assert w == "abyd"
+
+    def test_more_adding_fun(self):
+        s = 'a'.__add__('b') # s is a strbuf now
+        t = s + 'c'
+        u = s + 'd'
+        v = s + 'e'
+        assert v == 'abe'
+        assert u == 'abd'
+        assert t == 'abc'
+
+    def test_buh_even_more(self):
+        a = 'a'.__add__('b')
+        b = a + 'c'
+        c = '0'.__add__('1')
+        x = c + a
+        assert x == '01ab'
+
+    def test_add_non_string(self):
+        a = 'a'
+        a += 'b'
+        raises(TypeError, "a += 5")

From pypy.commits at gmail.com  Sat May 20 16:59:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 20 May 2017 13:59:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5920ae2e.c20f1c0a.3891d.34d7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91355:64deec640157
Date: 2017-05-20 21:58 +0100
http://bitbucket.org/pypy/pypy/changeset/64deec640157/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,7 +26,7 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
 WIN64 = sys.platform == 'win32' and sys.maxsize == 2**63 - 1
 
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -93,14 +96,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -125,27 +123,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -155,7 +152,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -167,15 +164,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -187,7 +187,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -200,7 +200,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -221,10 +221,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -258,7 +256,7 @@
         if (sys.platform == 'win32' and isinstance(argument, int)
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -280,6 +278,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -46,8 +46,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
@@ -222,9 +223,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,20 @@
 
 Internal refactoring of buffers and memoryviews. Memoryviews will now be
 accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -690,7 +690,8 @@
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
-    for cpyname in '''PyMethodObject PyListObject PyLongObject'''.split():
+    for cpyname in '''PyMethodObject PyListObject PyLongObject
+                      PyBaseExceptionObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
@@ -1613,7 +1614,6 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null, convert_result):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
 
     if w_dict is None:
         return 0
-
+    if not space.isinstance_w(w_dict, space.w_dict):
+        return 0 
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         Py_IncRef(space, py_dict.c__tmpkeys)
     else:
+        if not py_dict.c__tmpkeys:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
         w_keys = from_ref(space, py_dict.c__tmpkeys)
     ppos[0] += 1
     if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +229,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, widen(limit))
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,9 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -19,59 +20,68 @@
     PySequence_SetItem()  or expose the object to Python code before
     setting all items to a real object with PyList_SetItem().
     """
-    return space.newlist([None] * len)
+    w_list = space.newlist([None] * len)
+    #w_list.convert_to_cpy_strategy(space)
+    return w_list
 
- at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
-             result_borrowed=True)
-def PyList_SET_ITEM(space, w_list, index, w_item):
-    """Macro form of PyList_SetItem() without error checking. This is normally
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    assert isinstance(w_list, W_ListObject)
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
+def PyList_SET_ITEM(space, w_list, index, py_item):
+    """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
-    This function "steals" a reference to item, and, unlike PyList_SetItem(),
-    does not discard a reference to any item that it being replaced; any
-    reference in list at position i will be leaked.
+    "steals" a reference to item, and, unlike PyList_SetItem(), does not
+    discard a reference to any item that it being replaced; any reference in
+    list at position i will be leaked.
     """
-    assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    # Deliberately leak, so that it can be safely decref'd.
-    make_ref(space, w_list.getitem(index))
-    Py_DecRef(space, w_item)
-    w_list.setitem(index, w_item)
-    return w_item
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
-    Py_DecRef(space, w_item)
     if not isinstance(w_list, W_ListObject):
+        decref(space, w_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
+        decref(space, w_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.setitem(index, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
-    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    if w_list.strategy is not cpy_strategy:
-        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -30,8 +30,7 @@
         return subtype_dealloc.api_func
 
     def allocate(self, space, w_type, itemcount=0):
-        # similar to PyType_GenericAlloc?
-        # except that it's not related to any pypy object.
+        # typically called from PyType_GenericAlloc via typedescr.allocate
         # this returns a PyObject with ob_refcnt == 1.
 
         pytype = as_pyobj(space, w_type)
@@ -250,6 +249,8 @@
     w_obj = rawrefcount.to_obj(W_Root, pyobj)
     return w_obj is not None and w_obj is not w_marker_deallocating
 
+def w_obj_has_pyobj(w_obj):
+    return bool(rawrefcount.from_obj(PyObject, w_obj))
 
 def is_pyobj(x):
     if x is None or isinstance(x, W_Root):
@@ -275,13 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
+        at_least = rawrefcount.REFCNT_FROM_PYPY
     if pyobj:
-        assert pyobj.c_ob_refcnt > 0
+        assert pyobj.c_ob_refcnt >= at_least
         pyobj.c_ob_refcnt += 1
-        if not is_pyobj(obj):
-            keepalive_until_here(obj)
+        keepalive_until_here(obj)
     return pyobj
 
 
@@ -315,9 +317,14 @@
         obj = rffi.cast(PyObject, obj)
         if obj:
             assert obj.c_ob_refcnt > 0
+            assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            #else:
+            #    w_obj = rawrefcount.to_obj(W_Root, ref)
+            #    if w_obj is not None:
+            #        assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
     else:
         get_w_obj_and_decref(space, obj)
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,7 +14,7 @@
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
@@ -256,7 +256,7 @@
     check_num_args(space, w_args, 1)
     args_w = space.fixedview(w_args)
     index = space.int_w(space.index(args_w[0]))
-    null = lltype.nullptr(PyObject.TO)
+    null = rffi.cast(PyObject, 0)
     res = generic_cpy_call(space, func_target, w_self, index, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
@@ -285,7 +285,8 @@
     func_target = rffi.cast(objobjargproc, func)
     check_num_args(space, w_args, 1)
     w_key, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, None)
+    null = rffi.cast(PyObject, 0)
+    res = generic_cpy_call(space, func_target, w_self, w_key, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.w_None
@@ -611,6 +612,8 @@
             handled = True
 
     for tp_name, attr in [('tp_hash', '__hash__'),
+                          ('tp_as_sequence.c_sq_length', '__len__'),
+                          ('tp_as_mapping.c_mp_length', '__len__'),
                          ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -636,7 +639,8 @@
                           ('tp_as_number.c_nb_xor', '__xor__'),
                           ('tp_as_number.c_nb_or', '__or__'),
                           ('tp_as_sequence.c_sq_concat', '__add__'),
-                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+                          ('tp_as_mapping.c_mp_subscript', '__getitem__'),
                           ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -650,7 +654,7 @@
             handled = True
 
     # binary-with-Py_ssize_t-type
-    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -679,7 +683,48 @@
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
             handled = True
+    # ternary-with-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
 
+            @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, w_arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, w_arg1)
+                return 0
+            handled = True
+    # ternary-Py_size_t-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
+
+            @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, space.newint(arg1))
+                return 0
+            handled = True
     if handled:
         pass
     elif name == 'tp_setattro':
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -483,29 +483,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=-1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyObject], rffi.INT_real, error=-1)
 def Py_ReprEnter(space, object):
     """Called at the beginning of the tp_repr implementation to
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2330,6 +2330,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -2345,6 +2346,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -260,4 +260,60 @@
             ])
         d = module.get_type_dict(1)
         assert d['real'].__get__(1, 1) == 1
-
+    def test_advanced(self):
+        module = self.import_extension('foo', [
+            ("dict_len", "METH_O",
+            '''
+                int ret = args->ob_type->tp_as_mapping->mp_length(args);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_setitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 3 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1),
+                        PyTuple_GetItem(args, 2));
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_delitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 2 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1), NULL);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_next", "METH_VARARGS",
+            '''
+                PyObject *key, *value;
+                PyObject *arg = NULL;
+                Py_ssize_t pos = 0;
+                int ret = 0;
+                if ((PyArg_ParseTuple(args, "|O", &arg))) {
+                    if (arg && PyDict_Check(arg)) {
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                        /* test no crash if pos is not reset to 0*/
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                    }
+                }
+                return PyLong_FromLong(ret);
+            '''),
+            ])
+        d = {'a': 1, 'b':2}
+        assert module.dict_len(d) == 2
+        assert module.dict_setitem(d, 'a', 'c') == 0
+        assert d['a'] == 'c'
+        assert module.dict_delitem(d, 'a') == 0
+        r = module.dict_next({'a': 1, 'b': 2})
+        assert r == 2
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -339,3 +339,30 @@
                 def nested_flags():
                     return module.get_flags()""", ns)
         assert ns['nested_flags']() == (0, 0)
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,10 +38,12 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
 
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
@@ -152,29 +154,35 @@
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
-            ("test_macro_invocations", "METH_NOARGS",
+            ("test_macro_invocations", "METH_O",
              """
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
+             PyList_SET_ITEM(o, 0, args);
+             PyList_SET_ITEM(l, 1, args);
 
-             Py_INCREF(o);
-             PyList_SET_ITEM(o, 0, o);
-             Py_INCREF(o);
-             PyList_SET_ITEM(l, 1, o);
+             if(PyList_GET_ITEM(o, 0) != PyList_GET_ITEM(l, 1))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_ITEM error");
+                return NULL;
+             }
 
-             PyList_GET_ITEM(o, 0);
-             PyList_GET_ITEM(l, 1);
-
-             PyList_GET_SIZE(o);
-             PyList_GET_SIZE(l);
+             if(PyList_GET_SIZE(o) != PyList_GET_SIZE(l))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_SIZE error");
+                return NULL;
+             }
 
              return o;
              """
             )
         ])
-        x = module.test_macro_invocations()
-        assert x[0] is x[1] is x
+        s = 'abcdef'
+        x = module.test_macro_invocations(s)
+        assert x[0] is x[1] is s
 
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
@@ -183,39 +191,96 @@
                 PyObject* o, *o2, *o3;
                 o = PyList_New(1);
 
-                o2 = PyLong_FromLong(0);
+                o2 = PyBytes_FromString("test_get_item0");
+                Py_INCREF(o2);
                 PyList_SET_ITEM(o, 0, o2);
-                o2 = NULL;
 
                 o3 = PyList_GET_ITEM(o, 0);
                 Py_INCREF(o3);
-                Py_CLEAR(o);
+                Py_DECREF(o);
+                Py_DECREF(o2);
                 return o3;
              """)])
-        assert module.test_get_item() == 0
+        assert module.test_get_item() == b'test_get_item0'
 
-    def test_set_item_macro(self):
+    def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff_after_setitem", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
-                PyObject* o = PyList_New(0);
-                PyObject* o2 = PyList_New(0);
-                Py_ssize_t refcount, new_refcount;
+                /* test that the refcount differences for functions
+                 * are correct. diff1 - expected refcount diff for i1,
+                 *              diff2 - expected refcount diff for i2
+                 */
+                #define CHECKCOUNT(diff1, diff2, action) \
+                    new_count1 = Py_REFCNT(i1); \
+                    new_count2 = Py_REFCNT(i2); \
+                    diff = new_count1 - old_count1; \
+                    if (diff != diff1) {\
+                        sprintf(errbuffer, action \
+                            " i1 expected diff of %ld got %ld", (long)diff1, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    diff = new_count2 - old_count2; \
+                    if (diff != diff2) {\
+                        sprintf(errbuffer, action \
+                            " i2 expected diff of %ld got %ld", (long)diff2, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    old_count1 = new_count1; \
+                    old_count2 = new_count2;
 
-                PyList_Append(o, o2);  // does not steal o2
+                PyObject* tmp, *o = PyList_New(0);
+                char errbuffer[1024];
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
+                Py_ssize_t old_count1, new_count1;
+                Py_ssize_t old_count2, new_count2;
+                Py_ssize_t diff;
+                int ret;
 
-                refcount = Py_REFCNT(o2);
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
-                // Steal a reference to o2, but leak the old reference to o2.
-                // The net result should be no change in refcount.
-                PyList_SET_ITEM(o, 0, o2);
+                ret = PyList_Append(o, i1);
+                if (ret != 0) 
+                    return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(1, 0, "PyList_Append");
 
-                new_refcount = Py_REFCNT(o2);
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
 
-                Py_CLEAR(o);
-                Py_DECREF(o2); // append incref'd.
-                // Py_CLEAR(o2);  // naive implementation would fail here.
-                return PyLong_FromSsize_t(new_refcount - refcount);
+                PyList_SET_ITEM(o, 0, i2);
+                CHECKCOUNT(0, 0, "PyList_SET_ITEM");
+
+                tmp = PyList_GET_ITEM(o, 0);
+                if (tmp != i2)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(0, 0, "PyList_GET_ITEM");
+
+                PyList_SetItem(o, 0, i1);
+                CHECKCOUNT(0, -1, "PyList_Set_Item");
+
+                PyList_GetItem(o, 0);
+                CHECKCOUNT(0, 0, "PyList_Get_Item");
+
+                Py_DECREF(o); 
+                #ifndef PYPY_VERSION
+                {
+                    // PyPy deletes only at teardown
+                    CHECKCOUNT(-1, 0, "Py_DECREF(o)");
+                }
+                #endif
+                return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff_after_setitem() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -23,7 +23,7 @@
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
+    PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, Py_DecRef, as_pyobj)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -730,7 +730,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+ at cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
diff --git a/pypy/module/test_lib_pypy/README.txt b/pypy/module/test_lib_pypy/README.txt
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/README.txt
@@ -0,0 +1,4 @@
+This directory contains app-level tests are supposed to be run *after*
+translation. So you run them by saying:
+
+pypy pytest.py <testfile.py>
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/pypy/module/test_lib_pypy/ctypes_tests/README
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/README
@@ -0,0 +1,8 @@
+-------Ctypes tests------
+
+Unlike the other tests in the PyPy sources, these tests are assumed to run after the translation is complete.
+Therefore, using the resulting binary, you can then call, for example:
+
+/path/to/your_modified_pypy_binary pytest.py pypy/module/test_lib_pypy/ctypes_tests/test_libc.py
+
+This also applies to any other test in ctypes_tests.
\ No newline at end of file
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py b/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from ctypes import CDLL, POINTER, pointer, c_byte, c_int, c_char_p, CFUNCTYPE, c_void_p, c_size_t
-import sys
-import py
-from support import BaseCTypesTestChecker
-
-class MyCDLL(CDLL):
-    def __getattr__(self, attr):
-        fn = self[attr] # this way it's not cached as an attribute
-        fn._slowpath_allowed = False
-        return fn
-
-def setup_module(mod):
-    import conftest
-    _ctypes_test = str(conftest.sofile)
-    mod.dll = MyCDLL(_ctypes_test)  # slowpath not allowed
-    mod.dll2 = CDLL(_ctypes_test)   # slowpath allowed
-
-
-class TestFastpath(BaseCTypesTestChecker):
-
-    def test_fastpath_forbidden(self):
-        def myfunc():
-            pass
-        #
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        #
-        # so far, it's still using the slowpath
-        assert not tf_b._is_fastpath
-        tf_b.callable = myfunc
-        tf_b.argtypes = (c_byte,)
-        # errcheck prevented the fastpath to kick in
-        assert not tf_b._is_fastpath
-        #
-        del tf_b.callable
-        tf_b.argtypes = (c_byte,) # try to re-enable the fastpath
-        assert tf_b._is_fastpath
-        #
-        assert not tf_b._slowpath_allowed
-        py.test.raises(AssertionError, "tf_b.callable = myfunc")
-        py.test.raises(AssertionError, "tf_b('aaa')") # force a TypeError
-
-    def test_simple_args(self):
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        assert tf_b(-126) == -42
-
-    def test_from_cfunctype(self):
-        from _ctypes import _memmove_addr
-        functype = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)
-        my_memmove = functype(_memmove_addr)
-        assert my_memmove._is_fastpath
-
-    def test_undeclared_restype(self):
-        # make sure we get a fresh function
-        try:
-            del dll.tf_i
-        except AttributeError:
-            pass
-        tf_i = dll.tf_i
-        assert not tf_i._is_fastpath
-        tf_i.argtypes = (c_int,)
-        assert tf_i._is_fastpath
-        assert tf_i(12) == 4
-
-    def test_pointer_args(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-        v = c_int(42)
-        result = f(pointer(v))
-        assert type(result) == POINTER(c_int)
-        assert result.contents.value == 42
-
-    def test_simple_pointer_args(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        mystr = c_char_p("abcd")
-        result = f(mystr, ord("b"))
-        assert result == "bcd"
-
-    def test_strings(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        result = f("abcd", ord("b"))
-        assert result == "bcd"
-
-    def test_errcheck(self):
-        def errcheck(result, func, args):
-            return 'hello'
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.errcheck = errcheck
-        assert tf_b(-126) == 'hello'
-
-    def test_array_to_ptr(self):
-        ARRAY = c_int * 8
-        func = dll._testfunc_ai8
-        func.restype = POINTER(c_int)
-        func.argtypes = [ARRAY]
-        array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8)
-        ptr = func(array)
-        assert ptr[0] == 1
-        assert ptr[7] == 8
-
-
-class TestFallbackToSlowpath(BaseCTypesTestChecker):
-
-    def test_argtypes_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_char_p,)  # this is intentionally wrong
-        tf_b.argtypes = None # kill the fast path
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -42
-
-    def test_callable_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.callable = lambda x: x+1
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -125
-        tf_b.callable = None
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -478,7 +478,7 @@
         raises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
 
     def test_argument_conversion_and_checks(self):
-        py.test.skip("XXX currently broken on PyPy, sorry")
+        #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
         strlen = dll.my_strchr
         strlen.argtypes = [c_char_p, c_int]
         strlen.restype = c_char_p
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -580,55 +580,31 @@
         return space.newint(x)
 
     def descr_eq(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -637,18 +613,6 @@
 
     _StringMethods_descr_add = descr_add
     def descr_add(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods_descr_join = descr_join
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -229,15 +229,13 @@
         return list(items)
 
     def switch_to_object_strategy(self):
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        if self.strategy is object_strategy:
+            return
         list_w = self.getitems()
-        object_strategy = self.space.fromcache(ObjectListStrategy)
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -18,7 +18,7 @@
 # Object imports
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -82,9 +82,6 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
-        if self.config.objspace.std.withstrbuf:
-            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
-
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/strbufobject.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import SimpleView, StringBuffer
-from pypy.interpreter.error import OperationError
-from rpython.rlib.rstring import StringBuilder
-
-
-class W_StringBufferObject(W_AbstractBytesObject):
-    w_str = None
-
-    def __init__(self, builder):
-        self.builder = builder             # StringBuilder
-        self.length = builder.getlength()
-
-    def force(self):
-        if self.w_str is None:
-            s = self.builder.build()
-            if self.length < len(s):
-                s = s[:self.length]
-            self.w_str = W_BytesObject(s)
-            return s
-        else:
-            return self.w_str._value
-
-    def __repr__(self):
-        """ representation for debugging purposes """
-        return "%s(%r[:%d])" % (
-            self.__class__.__name__, self.builder, self.length)
-
-    def unwrap(self, space):
-        return self.force()
-
-    def bytes_w(self, space):
-        return self.force()
-
-    def buffer_w(self, space, flags):
-        return SimpleView(StringBuffer(self.force()))
-
-    def descr_len(self, space):
-        return space.newint(self.length)
-
-    def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
-        else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
-
-    def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
-        return self
-
-
-delegation_dict = {}
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
-
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
-
-W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -62,13 +62,6 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
-    def test_withstrbuf_fastpath_isinstance(self):
-        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
-
-        space = gettestobjspace(withstrbuf=True)
-        cls = space._get_interplevel_cls(space.w_bytes)
-        assert cls is W_AbstractBytesObject
-
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import py
-
-from pypy.objspace.std.test import test_bytesobject
-
-class AppTestStringObject(test_bytesobject.AppTestBytesObject):
-    spaceconfig = {"objspace.std.withstrbuf": True}
-
-    def test_basic(self):
-        import __pypy__
-        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
-        # away on AST level
-        s = b"Hello, ".__add__(b"World!")
-        assert type(s) is bytes
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-
-    def test_add_twice(self):
-        x = b"a".__add__(b"b")
-        y = x + b"c"
-        c = x + b"d"
-        assert y == b"abc"
-        assert c == b"abd"
-
-    def test_add(self):
-        import __pypy__
-        all = b""
-        for i in range(20):
-            all += str(i).encode()
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
-        assert all == b"012345678910111213141516171819"
-
-    def test_hash(self):
-        import __pypy__
-        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
-        t = b'a' * 101
-        s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-        assert hash(s) == hash(t)
-
-    def test_len(self):
-        s = b"a".__add__(b"b")
-        r = b"c".__add__(b"d")
-        t = s + r
-        assert len(s) == 2
-        assert len(r) == 2
-        assert len(t) == 4
-
-    def test_buffer(self):
-        s = b'a'.__add__(b'b')
-        assert memoryview(s) == b'ab'
-
-    def test_add_strbuf(self):
-        # make three strbuf objects
-        s = b'a'.__add__(b'b')
-        t = b'x'.__add__(b'c')
-        u = b'y'.__add__(b'd')
-
-        # add two different strbufs to the same string
-        v = s + t
-        w = s + u
-
-        # check that insanity hasn't resulted.
-        assert v == b"abxc"
-        assert w == b"abyd"
-
-    def test_more_adding_fun(self):
-        s = b'a'.__add__(b'b') # s is a strbuf now
-        t = s + b'c'
-        u = s + b'd'
-        v = s + b'e'
-        assert v == b'abe'
-        assert u == b'abd'
-        assert t == b'abc'
-
-    def test_buh_even_more(self):
-        a = b'a'.__add__(b'b')
-        b = a + b'c'
-        c = b'0'.__add__(b'1')
-        x = c + a
-        assert x == b'01ab'
-
-    def test_add_non_string(self):
-        a = b'a'
-        a += b'b'
-        raises(TypeError, "a += 5")
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -239,7 +239,7 @@
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
         includes.append('sys/sysmacros.h')
-    if sys.platform.startswith('freebsd'):
+    if sys.platform.startswith('freebsd') or sys.platform.startswith('openbsd'):
         includes.append('sys/ttycom.h')
     libraries = ['util']
 eci = ExternalCompilationInfo(
diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c
--- a/rpython/rlib/rvmprof/src/shared/machine.c
+++ b/rpython/rlib/rvmprof/src/shared/machine.c
@@ -4,6 +4,7 @@
 #include <stdio.h>
 
 #ifdef VMPROF_UNIX
+#include <string.h>
 #include <unistd.h>
 #include <fcntl.h>
 #endif
diff --git a/rpython/rlib/streamio.py b/rpython/rlib/streamio.py
--- a/rpython/rlib/streamio.py
+++ b/rpython/rlib/streamio.py
@@ -902,18 +902,30 @@
         self.do_read = base.read
         self.do_write = base.write
         self.do_flush = base.flush_buffers
-        self.lfbuffer = ""
+        self.readahead_count = 0   # either 0 or 1
 
     def read(self, n=-1):
-        data = self.lfbuffer + self.do_read(n)
-        self.lfbuffer = ""
+        """If n >= 1, this should read between 1 and n bytes."""
+        if n <= 0:
+            if n < 0:
+                return self.readall()
+            else:
+                return ""
+
+        data = self.do_read(n - self.readahead_count)
+        if self.readahead_count > 0:
+            data = self.readahead_char + data
+            self.readahead_count = 0
+
         if data.endswith("\r"):
             c = self.do_read(1)
-            if c and c[0] == '\n':
-                data = data + '\n'
-                self.lfbuffer = c[1:]
-            else:
-                self.lfbuffer = c
+            if len(c) >= 1:
+                assert len(c) == 1
+                if c[0] == '\n':
+                    data = data + '\n'
+                else:
+                    self.readahead_char = c[0]
+                    self.readahead_count = 1
 
         result = []
         offset = 0
@@ -936,21 +948,21 @@
 
     def tell(self):
         pos = self.base.tell()
-        return pos - len(self.lfbuffer)
+        return pos - self.readahead_count
 
     def seek(self, offset, whence):
         if whence == 1:
-            offset -= len(self.lfbuffer)   # correct for already-read-ahead character
+            offset -= self.readahead_count   # correct for already-read-ahead character
         self.base.seek(offset, whence)
-        self.lfbuffer = ""
+        self.readahead_count = 0
 
     def flush_buffers(self):
-        if self.lfbuffer:
+        if self.readahead_count > 0:
             try:
-                self.base.seek(-len(self.lfbuffer), 1)
+                self.base.seek(-self.readahead_count, 1)
             except (MyNotImplementedError, OSError):
                 return
-            self.lfbuffer = ""
+            self.readahead_count = 0
         self.do_flush()
 
     def write(self, data):
diff --git a/rpython/rlib/test/test_streamio.py b/rpython/rlib/test/test_streamio.py
--- a/rpython/rlib/test/test_streamio.py
+++ b/rpython/rlib/test/test_streamio.py
@@ -657,6 +657,23 @@
             assert line == ''
         self.interpret(f, [])
 
+    def test_read1(self):
+        s_input = "abc\r\nabc\nd\r\nef\r\ngha\rbc\rdef\n\r\n\r"
+        s_output = "abc\nabc\nd\nef\ngha\rbc\rdef\n\n\r"
+        assert s_output == s_input.replace('\r\n', '\n')
+        packets = list(s_input)
+        expected = list(s_output)
+        crlf = streamio.TextCRLFFilter(TSource(packets))
+        def f():
+            blocks = []
+            while True:
+                block = crlf.read(1)
+                if not block:
+                    break
+                blocks.append(block)
+            assert blocks == expected
+        self.interpret(f, [])
+
 class TestTextCRLFFilterLLInterp(BaseTestTextCRLFFilter):
     pass
 

From pypy.commits at gmail.com  Sat May 20 17:07:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 20 May 2017 14:07:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix translation
Message-ID: <5920b014.493a1c0a.30103.df8b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91356:c971f0f22925
Date: 2017-05-20 23:45 +0300
http://bitbucket.org/pypy/pypy/changeset/c971f0f22925/

Log:	fix translation

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -27,7 +27,6 @@
 @always_inline
 def get_list_storage(space, w_list):
     from pypy.module.cpyext.sequence import CPyListStrategy
-    assert isinstance(w_list, W_ListObject)
     w_list.convert_to_cpy_strategy(space)
     return CPyListStrategy.unerase(w_list.lstorage)
 
@@ -40,6 +39,7 @@
     discard a reference to any item that it being replaced; any reference in
     list at position i will be leaked.
     """
+    assert isinstance(w_list, W_ListObject)
     storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
     storage._elems[index] = py_item
@@ -53,10 +53,10 @@
     an item already in the list at the affected position.
     """
     if not isinstance(w_list, W_ListObject):
-        decref(space, w_item)
+        decref(space, py_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
-        decref(space, w_item)
+        decref(space, py_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
     storage = get_list_storage(space, w_list)
     py_old = storage._elems[index]
@@ -66,6 +66,7 @@
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GET_ITEM(space, w_list, index):
+    assert isinstance(w_list, W_ListObject)
     storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
     return storage._elems[index]     # borrowed ref

From pypy.commits at gmail.com  Sat May 20 19:58:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 20 May 2017 16:58:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5920d822.519c1c0a.a9924.d648@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91357:709ffd0a77bc
Date: 2017-05-21 00:57 +0100
http://bitbucket.org/pypy/pypy/changeset/709ffd0a77bc/

Log:	hg merge default

diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -27,7 +27,6 @@
 @always_inline
 def get_list_storage(space, w_list):
     from pypy.module.cpyext.sequence import CPyListStrategy
-    assert isinstance(w_list, W_ListObject)
     w_list.convert_to_cpy_strategy(space)
     return CPyListStrategy.unerase(w_list.lstorage)
 
@@ -40,6 +39,7 @@
     discard a reference to any item that it being replaced; any reference in
     list at position i will be leaked.
     """
+    assert isinstance(w_list, W_ListObject)
     storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
     storage._elems[index] = py_item
@@ -53,10 +53,10 @@
     an item already in the list at the affected position.
     """
     if not isinstance(w_list, W_ListObject):
-        decref(space, w_item)
+        decref(space, py_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
-        decref(space, w_item)
+        decref(space, py_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
     storage = get_list_storage(space, w_list)
     py_old = storage._elems[index]
@@ -66,6 +66,7 @@
 
 @cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GET_ITEM(space, w_list, index):
+    assert isinstance(w_list, W_ListObject)
     storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
     return storage._elems[index]     # borrowed ref

From pypy.commits at gmail.com  Sun May 21 05:36:12 2017
From: pypy.commits at gmail.com (antocuni)
Date: Sun, 21 May 2017 02:36:12 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix a misuse of box.getfloat()
 vs box.getfloatstorage(): this was harmless on 64 bit of course,
 but caused test_llop:test_gc_store_indexed_double to fail on 32 bit
Message-ID: <59215f8c.9ba3df0a.17259.3dff@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91358:0e91701e92f0
Date: 2017-05-21 11:35 +0200
http://bitbucket.org/pypy/pypy/changeset/0e91701e92f0/

Log:	fix a misuse of box.getfloat() vs box.getfloatstorage(): this was
	harmless on 64 bit of course, but caused
	test_llop:test_gc_store_indexed_double to fail on 32 bit

diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -261,7 +261,7 @@
     if arraydescr.is_array_of_pointers():
         raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
     elif arraydescr.is_array_of_floats():
-        floatval = valuebox.getfloat()
+        floatval = valuebox.getfloatstorage()
         cpu.bh_gc_store_indexed_f(addr, index, floatval, scale, base_ofs, bytes,
                                   arraydescr)
     else:

From pypy.commits at gmail.com  Mon May 22 03:44:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 May 2017 00:44:22 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: test,
 fix for cpytext hash of empty string subclass not returning -1
Message-ID: <592296d6.153f1c0a.b575c.4f09@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91360:c2f5f5816876
Date: 2017-05-22 10:43 +0300
http://bitbucket.org/pypy/pypy/changeset/c2f5f5816876/

Log:	test, fix for cpytext hash of empty string subclass not returning -1

diff --git a/pypy/module/cpyext/test/test_bytesobject.py b/pypy/module/cpyext/test/test_bytesobject.py
--- a/pypy/module/cpyext/test/test_bytesobject.py
+++ b/pypy/module/cpyext/test/test_bytesobject.py
@@ -452,6 +452,7 @@
         assert 3 == module.get_len(a)
         b = module.newsubstr('')
         assert 0 == module.get_len(b)
+        assert hash(b) in [0, -2]
 
 class TestBytes(BaseApiTest):
     def test_bytes_resize(self, space):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -366,7 +366,6 @@
         add_tp_new_wrapper(space, dict_w, pto)
     if not pto.c_tp_hash:
         dict_w['__hash__'] = space.w_None
-    print 'pto', rffi.charp2str(pto.c_tp_name), 'c_tp_hash', pto.c_tp_hash, dict_w['__hash__']
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -592,6 +592,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1)
         return space.newint(x)
 
     def descr_format(self, space, __args__):
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -744,6 +744,7 @@
         # disabled: assert hash('') == 0 --- different special case
         assert hash('hello') & 0x7fffffff == 0x347697fd
         assert hash('hello world!') & 0x7fffffff == 0x2f0bb411
+        assert hash('') in [0, -2]
 
     def test_buffer(self):
         x = b"he"

From pypy.commits at gmail.com  Mon May 22 03:44:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 May 2017 00:44:20 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: merge default into branch
Message-ID: <592296d4.0d2e1c0a.8c280.69f5@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91359:21f235773e6d
Date: 2017-05-21 00:07 +0300
http://bitbucket.org/pypy/pypy/changeset/21f235773e6d/

Log:	merge default into branch

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -220,9 +220,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,20 @@
 
 Internal refactoring of buffers and memoryviews. Memoryviews will now be
 accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1557,7 +1557,6 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +229,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, widen(limit))
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,9 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -19,59 +20,69 @@
     PySequence_SetItem()  or expose the object to Python code before
     setting all items to a real object with PyList_SetItem().
     """
-    return space.newlist([None] * len)
+    w_list = space.newlist([None] * len)
+    #w_list.convert_to_cpy_strategy(space)
+    return w_list
 
- at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
-             result_borrowed=True)
-def PyList_SET_ITEM(space, w_list, index, w_item):
-    """Macro form of PyList_SetItem() without error checking. This is normally
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
+def PyList_SET_ITEM(space, w_list, index, py_item):
+    """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
-    This function "steals" a reference to item, and, unlike PyList_SetItem(),
-    does not discard a reference to any item that it being replaced; any
-    reference in list at position i will be leaked.
+    "steals" a reference to item, and, unlike PyList_SetItem(), does not
+    discard a reference to any item that it being replaced; any reference in
+    list at position i will be leaked.
     """
     assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    # Deliberately leak, so that it can be safely decref'd.
-    make_ref(space, w_list.getitem(index))
-    Py_DecRef(space, w_item)
-    w_list.setitem(index, w_item)
-    return w_item
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
-    Py_DecRef(space, w_item)
     if not isinstance(w_list, W_ListObject):
+        decref(space, py_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
+        decref(space, py_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.setitem(index, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
-    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    if w_list.strategy is not cpy_strategy:
-        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -30,8 +30,7 @@
         return subtype_dealloc.api_func
 
     def allocate(self, space, w_type, itemcount=0):
-        # similar to PyType_GenericAlloc?
-        # except that it's not related to any pypy object.
+        # typically called from PyType_GenericAlloc via typedescr.allocate
         # this returns a PyObject with ob_refcnt == 1.
 
         pytype = as_pyobj(space, w_type)
@@ -250,6 +249,8 @@
     w_obj = rawrefcount.to_obj(W_Root, pyobj)
     return w_obj is not None and w_obj is not w_marker_deallocating
 
+def w_obj_has_pyobj(w_obj):
+    return bool(rawrefcount.from_obj(PyObject, w_obj))
 
 def is_pyobj(x):
     if x is None or isinstance(x, W_Root):
@@ -275,13 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
+        at_least = rawrefcount.REFCNT_FROM_PYPY
     if pyobj:
-        assert pyobj.c_ob_refcnt > 0
+        assert pyobj.c_ob_refcnt >= at_least
         pyobj.c_ob_refcnt += 1
-        if not is_pyobj(obj):
-            keepalive_until_here(obj)
+        keepalive_until_here(obj)
     return pyobj
 
 
@@ -315,9 +317,14 @@
         obj = rffi.cast(PyObject, obj)
         if obj:
             assert obj.c_ob_refcnt > 0
+            assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            #else:
+            #    w_obj = rawrefcount.to_obj(W_Root, ref)
+            #    if w_obj is not None:
+            #        assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
     else:
         get_w_obj_and_decref(space, obj)
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -490,29 +490,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyFileObject], lltype.Void)
 def PyFile_IncUseCount(space, p):
     """Increments the PyFileObject's internal use count to indicate
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -1872,6 +1872,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -1887,6 +1888,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -331,3 +331,30 @@
                 def nested_flags():
                     return module.get_flags()""" in ns
         assert ns['nested_flags']() == (1, 0x2000)  # CO_FUTURE_DIVISION
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,11 +38,13 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
-    
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
+
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
@@ -152,29 +154,35 @@
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
-            ("test_macro_invocations", "METH_NOARGS",
+            ("test_macro_invocations", "METH_O",
              """
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
+             PyList_SET_ITEM(o, 0, args);
+             PyList_SET_ITEM(l, 1, args);
 
-             Py_INCREF(o);
-             PyList_SET_ITEM(o, 0, o);
-             Py_INCREF(o);
-             PyList_SET_ITEM(l, 1, o);
+             if(PyList_GET_ITEM(o, 0) != PyList_GET_ITEM(l, 1))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_ITEM error");
+                return NULL;
+             }
 
-             PyList_GET_ITEM(o, 0);
-             PyList_GET_ITEM(l, 1);
-
-             PyList_GET_SIZE(o);
-             PyList_GET_SIZE(l);
+             if(PyList_GET_SIZE(o) != PyList_GET_SIZE(l))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_SIZE error");
+                return NULL;
+             }
 
              return o;
              """
             )
         ])
-        x = module.test_macro_invocations()
-        assert x[0] is x[1] is x
+        s = 'abcdef'
+        x = module.test_macro_invocations(s)
+        assert x[0] is x[1] is s
 
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
@@ -183,39 +191,96 @@
                 PyObject* o, *o2, *o3;
                 o = PyList_New(1);
 
-                o2 = PyInt_FromLong(0);
+                o2 = PyBytes_FromString("test_get_item0");
+                Py_INCREF(o2);
                 PyList_SET_ITEM(o, 0, o2);
-                o2 = NULL;
 
                 o3 = PyList_GET_ITEM(o, 0);
                 Py_INCREF(o3);
-                Py_CLEAR(o);
+                Py_DECREF(o);
+                Py_DECREF(o2);
                 return o3;
              """)])
-        assert module.test_get_item() == 0
+        assert module.test_get_item() == b'test_get_item0'
 
-    def test_set_item_macro(self):
+    def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff_after_setitem", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
-                PyObject* o = PyList_New(0);
-                PyObject* o2 = PyList_New(0);
-                Py_ssize_t refcount, new_refcount;
+                /* test that the refcount differences for functions
+                 * are correct. diff1 - expected refcount diff for i1,
+                 *              diff2 - expected refcount diff for i2
+                 */
+                #define CHECKCOUNT(diff1, diff2, action) \
+                    new_count1 = Py_REFCNT(i1); \
+                    new_count2 = Py_REFCNT(i2); \
+                    diff = new_count1 - old_count1; \
+                    if (diff != diff1) {\
+                        sprintf(errbuffer, action \
+                            " i1 expected diff of %ld got %ld", (long)diff1, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    diff = new_count2 - old_count2; \
+                    if (diff != diff2) {\
+                        sprintf(errbuffer, action \
+                            " i2 expected diff of %ld got %ld", (long)diff2, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    old_count1 = new_count1; \
+                    old_count2 = new_count2;
 
-                PyList_Append(o, o2);  // does not steal o2
+                PyObject* tmp, *o = PyList_New(0);
+                char errbuffer[1024];
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
+                Py_ssize_t old_count1, new_count1;
+                Py_ssize_t old_count2, new_count2;
+                Py_ssize_t diff;
+                int ret;
 
-                refcount = Py_REFCNT(o2);
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
-                // Steal a reference to o2, but leak the old reference to o2.
-                // The net result should be no change in refcount.
-                PyList_SET_ITEM(o, 0, o2);
+                ret = PyList_Append(o, i1);
+                if (ret != 0) 
+                    return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(1, 0, "PyList_Append");
 
-                new_refcount = Py_REFCNT(o2);
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
 
-                Py_CLEAR(o);
-                Py_DECREF(o2); // append incref'd.
-                // Py_CLEAR(o2);  // naive implementation would fail here.
-                return PyLong_FromSsize_t(new_refcount - refcount);
+                PyList_SET_ITEM(o, 0, i2);
+                CHECKCOUNT(0, 0, "PyList_SET_ITEM");
+
+                tmp = PyList_GET_ITEM(o, 0);
+                if (tmp != i2)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(0, 0, "PyList_GET_ITEM");
+
+                PyList_SetItem(o, 0, i1);
+                CHECKCOUNT(0, -1, "PyList_Set_Item");
+
+                PyList_GetItem(o, 0);
+                CHECKCOUNT(0, 0, "PyList_Get_Item");
+
+                Py_DECREF(o); 
+                #ifndef PYPY_VERSION
+                {
+                    // PyPy deletes only at teardown
+                    CHECKCOUNT(-1, 0, "Py_DECREF(o)");
+                }
+                #endif
+                return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff_after_setitem() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -24,7 +24,7 @@
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
+    PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, Py_DecRef, as_pyobj)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -611,55 +611,31 @@
         return mod_format(space, w_values, self, do_unicode=False)
 
     def descr_eq(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -677,18 +653,6 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -230,15 +230,13 @@
         return list(items)
 
     def switch_to_object_strategy(self):
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        if self.strategy is object_strategy:
+            return
         list_w = self.getitems()
-        object_strategy = self.space.fromcache(ObjectListStrategy)
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -81,9 +81,6 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
-        if self.config.objspace.std.withstrbuf:
-            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
-
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
@@ -285,7 +282,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/strbufobject.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import SimpleView, StringBuffer
-from pypy.interpreter.error import OperationError
-from rpython.rlib.rstring import StringBuilder
-
-
-class W_StringBufferObject(W_AbstractBytesObject):
-    w_str = None
-
-    def __init__(self, builder):
-        self.builder = builder             # StringBuilder
-        self.length = builder.getlength()
-
-    def force(self):
-        if self.w_str is None:
-            s = self.builder.build()
-            if self.length < len(s):
-                s = s[:self.length]
-            self.w_str = W_BytesObject(s)
-            return s
-        else:
-            return self.w_str._value
-
-    def __repr__(self):
-        """ representation for debugging purposes """
-        return "%s(%r[:%d])" % (
-            self.__class__.__name__, self.builder, self.length)
-
-    def unwrap(self, space):
-        return self.force()
-
-    def str_w(self, space):
-        return self.force()
-
-    def buffer_w(self, space, flags):
-        return SimpleView(StringBuffer(self.force()))
-
-    def descr_len(self, space):
-        return space.newint(self.length)
-
-    def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
-        else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
-
-    def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
-        return self
-
-
-delegation_dict = {}
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
-
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
-
-W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -57,13 +57,6 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
-    def test_withstrbuf_fastpath_isinstance(self):
-        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
-
-        space = gettestobjspace(withstrbuf=True)
-        cls = space._get_interplevel_cls(space.w_bytes)
-        assert cls is W_AbstractBytesObject
-
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import py
-
-from pypy.objspace.std.test import test_bytesobject
-
-class AppTestStringObject(test_bytesobject.AppTestBytesObject):
-    spaceconfig = {"objspace.std.withstrbuf": True}
-
-    def test_basic(self):
-        import __pypy__
-        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
-        # away on AST level
-        s = "Hello, ".__add__("World!")
-        assert type(s) is str
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-
-    def test_add_twice(self):
-        x = "a".__add__("b")
-        y = x + "c"
-        c = x + "d"
-        assert y == "abc"
-        assert c == "abd"
-
-    def test_add(self):
-        import __pypy__
-        all = ""
-        for i in range(20):
-            all += str(i)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
-        assert all == "012345678910111213141516171819"
-
-    def test_hash(self):
-        import __pypy__
-        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
-        t = 'a' * 101
-        s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-        assert hash(s) == hash(t)
-
-    def test_len(self):
-        s = "a".__add__("b")
-        r = "c".__add__("d")
-        t = s + r
-        assert len(s) == 2
-        assert len(r) == 2
-        assert len(t) == 4
-
-    def test_buffer(self):
-        s = b'a'.__add__(b'b')
-        assert buffer(s) == buffer(b'ab')
-        assert memoryview(s) == b'ab'
-
-    def test_add_strbuf(self):
-        # make three strbuf objects
-        s = 'a'.__add__('b')
-        t = 'x'.__add__('c')
-        u = 'y'.__add__('d')
-
-        # add two different strbufs to the same string
-        v = s + t
-        w = s + u
-
-        # check that insanity hasn't resulted.
-        assert v == "abxc"
-        assert w == "abyd"
-
-    def test_more_adding_fun(self):
-        s = 'a'.__add__('b') # s is a strbuf now
-        t = s + 'c'
-        u = s + 'd'
-        v = s + 'e'
-        assert v == 'abe'
-        assert u == 'abd'
-        assert t == 'abc'
-
-    def test_buh_even_more(self):
-        a = 'a'.__add__('b')
-        b = a + 'c'
-        c = '0'.__add__('1')
-        x = c + a
-        assert x == '01ab'
-
-    def test_add_non_string(self):
-        a = 'a'
-        a += 'b'
-        raises(TypeError, "a += 5")

From pypy.commits at gmail.com  Mon May 22 06:01:17 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 03:01:17 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: enforce the annotation of
 write_float_at_mem: the problem fixed by 0e91701e92f0 was that we mixed a
 r_floatstorage with a float: on 32 bit they are different types but the
 annotator union()ed them to Float,
 causing nonsense at every place calling write_float_at_mem. The goal of this
 commit is to catch this kind of bug at translation time
Message-ID: <5922b6ed.430d1c0a.4f511.7f4b@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91361:65b6be2c1b4f
Date: 2017-05-22 12:00 +0200
http://bitbucket.org/pypy/pypy/changeset/65b6be2c1b4f/

Log:	enforce the annotation of write_float_at_mem: the problem fixed by
	0e91701e92f0 was that we mixed a r_floatstorage with a float: on 32
	bit they are different types but the annotator union()ed them to
	Float, causing nonsense at every place calling write_float_at_mem.
	The goal of this commit is to catch this kind of bug at translation
	time

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -5,7 +5,8 @@
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
 from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
+from rpython.rlib.objectmodel import (we_are_translated, specialize, compute_hash,
+                                      enforceargs)
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -494,6 +495,7 @@
         return llop.raw_load(longlong.FLOATSTORAGE, gcref, ofs)
 
     @specialize.argtype(1)
+    @enforceargs(newvalue=longlong.r_float_storage)
     def write_float_at_mem(self, gcref, ofs, newvalue):
         llop.raw_store(lltype.Void, gcref, ofs, newvalue)
 

From pypy.commits at gmail.com  Mon May 22 09:25:11 2017
From: pypy.commits at gmail.com (ltratt)
Date: Mon, 22 May 2017 06:25:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Build main binaries on OpenBSD with
 wxallowed.
Message-ID: <5922e6b7.10d91c0a.63d9b.d8a7@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91362:008c6f3a2f8f
Date: 2017-05-22 15:18 +0200
http://bitbucket.org/pypy/pypy/changeset/008c6f3a2f8f/

Log:	Build main binaries on OpenBSD with wxallowed.

	On RPython, JIT compilers that need to read and write to the same
	page need to be marked as wxallowed (unless they've been buit to
	cope with this restriction). Previously, this meant specifying
	LDFLAGS in the environment before building an RPython VM which I
	always forgot to do. This commit automatically marks the final
	binary as wxallowed without any such annoyances.

diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -16,5 +16,14 @@
         libraries=set(libraries + ("intl", "iconv"))
         return ['-l%s' % lib for lib in libraries if lib not in ["crypt", "dl", "rt"]]
 
+    def makefile_link_flags(self):
+        # On OpenBSD, we need to build the final binary with the link flags
+        # below. However, if we modify self.link_flags to include these, the
+        # various platform check binaries that RPython builds end up with these
+        # flags: since these binaries are generally located on /tmp -- which
+        # isn't a wxallowed file system -- that gives rise to "permission
+        # denied" errors, which kill the build.
+        return list(self.link_flags) + ["-Wl,-z,wxneeded"]
+
 class OpenBSD_64(OpenBSD):
     shared_only = ('-fPIC',)
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -98,6 +98,9 @@
     def get_shared_only_compile_flags(self):
         return tuple(self.shared_only) + ('-fvisibility=hidden',)
 
+    def makefile_link_flags(self):
+        return list(self.link_flags)
+
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
                      no_precompile_cfiles = [], config=None):
@@ -113,7 +116,7 @@
         else:
             exe_name = exe_name.new(ext=self.exe_ext)
 
-        linkflags = list(self.link_flags)
+        linkflags = self.makefile_link_flags()
         if shared:
             linkflags = self._args_for_shared(linkflags)
 

From pypy.commits at gmail.com  Mon May 22 09:25:13 2017
From: pypy.commits at gmail.com (ltratt)
Date: Mon, 22 May 2017 06:25:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merge default
Message-ID: <5922e6b9.918bdf0a.4cfb5.f7fb@mx.google.com>

Author: Laurence Tratt <laurie at tratt.net>
Branch: 
Changeset: r91363:cfe544728497
Date: 2017-05-22 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/cfe544728497/

Log:	Merge default

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,9 +26,9 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
-WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+WIN64 = sys.platform == 'win32' and sys.maxint == 2 ** 63 - 1
 
 
 def get_com_error(errcode, riid, pIunk):
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -94,14 +97,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -126,27 +124,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -156,7 +153,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -168,15 +165,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -188,7 +188,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -201,7 +201,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -222,10 +222,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -259,7 +257,7 @@
         if (sys.platform == 'win32' and isinstance(argument, (int, long))
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -281,6 +279,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -317,7 +316,7 @@
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
-                print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
+                print >> sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
             if self._restype_ is not None:
                 return res
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -220,9 +220,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,20 @@
 
 Internal refactoring of buffers and memoryviews. Memoryviews will now be
 accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -677,7 +677,7 @@
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
     for cpyname in '''PyMethodObject PyListObject PyLongObject
-                      PyClassObject'''.split():
+                      PyClassObject PyBaseExceptionObject'''.split():
         FORWARD_DECLS.append('typedef struct { PyObject_HEAD } %s'
                              % (cpyname, ))
 build_exported_objects()
@@ -1557,7 +1557,6 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -257,7 +257,8 @@
 
     if w_dict is None:
         return 0
-
+    if not space.isinstance_w(w_dict, space.w_dict):
+        return 0 
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
@@ -268,6 +269,9 @@
         py_dict.c__tmpkeys = create_ref(space, w_keys)
         Py_IncRef(space, py_dict.c__tmpkeys)
     else:
+        if not py_dict.c__tmpkeys:
+            # pos should have been 0, cannot fail so return 0
+            return 0;
         w_keys = from_ref(space, py_dict.c__tmpkeys)
     ppos[0] += 1
     if pos >= space.len_w(w_keys):
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +229,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, widen(limit))
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,9 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -19,59 +20,69 @@
     PySequence_SetItem()  or expose the object to Python code before
     setting all items to a real object with PyList_SetItem().
     """
-    return space.newlist([None] * len)
+    w_list = space.newlist([None] * len)
+    #w_list.convert_to_cpy_strategy(space)
+    return w_list
 
- at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
-             result_borrowed=True)
-def PyList_SET_ITEM(space, w_list, index, w_item):
-    """Macro form of PyList_SetItem() without error checking. This is normally
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
+def PyList_SET_ITEM(space, w_list, index, py_item):
+    """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
-    This function "steals" a reference to item, and, unlike PyList_SetItem(),
-    does not discard a reference to any item that it being replaced; any
-    reference in list at position i will be leaked.
+    "steals" a reference to item, and, unlike PyList_SetItem(), does not
+    discard a reference to any item that it being replaced; any reference in
+    list at position i will be leaked.
     """
     assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    # Deliberately leak, so that it can be safely decref'd.
-    make_ref(space, w_list.getitem(index))
-    Py_DecRef(space, w_item)
-    w_list.setitem(index, w_item)
-    return w_item
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
-    Py_DecRef(space, w_item)
     if not isinstance(w_list, W_ListObject):
+        decref(space, py_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
+        decref(space, py_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.setitem(index, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
-    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    if w_list.strategy is not cpy_strategy:
-        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -30,8 +30,7 @@
         return subtype_dealloc.api_func
 
     def allocate(self, space, w_type, itemcount=0):
-        # similar to PyType_GenericAlloc?
-        # except that it's not related to any pypy object.
+        # typically called from PyType_GenericAlloc via typedescr.allocate
         # this returns a PyObject with ob_refcnt == 1.
 
         pytype = as_pyobj(space, w_type)
@@ -250,6 +249,8 @@
     w_obj = rawrefcount.to_obj(W_Root, pyobj)
     return w_obj is not None and w_obj is not w_marker_deallocating
 
+def w_obj_has_pyobj(w_obj):
+    return bool(rawrefcount.from_obj(PyObject, w_obj))
 
 def is_pyobj(x):
     if x is None or isinstance(x, W_Root):
@@ -275,13 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
+        at_least = rawrefcount.REFCNT_FROM_PYPY
     if pyobj:
-        assert pyobj.c_ob_refcnt > 0
+        assert pyobj.c_ob_refcnt >= at_least
         pyobj.c_ob_refcnt += 1
-        if not is_pyobj(obj):
-            keepalive_until_here(obj)
+        keepalive_until_here(obj)
     return pyobj
 
 
@@ -315,9 +317,14 @@
         obj = rffi.cast(PyObject, obj)
         if obj:
             assert obj.c_ob_refcnt > 0
+            assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            #else:
+            #    w_obj = rawrefcount.to_obj(W_Root, ref)
+            #    if w_obj is not None:
+            #        assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
     else:
         get_w_obj_and_decref(space, obj)
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -14,7 +14,7 @@
     ssizessizeargfunc, ssizeobjargproc, iternextfunc, initproc, richcmpfunc,
     cmpfunc, hashfunc, descrgetfunc, descrsetfunc, objobjproc, objobjargproc,
     readbufferproc, getbufferproc, releasebufferproc, ssizessizeobjargproc)
-from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj
+from pypy.module.cpyext.pyobject import make_ref, decref, as_pyobj, from_ref
 from pypy.module.cpyext.pyerrors import PyErr_Occurred
 from pypy.module.cpyext.memoryobject import fill_Py_buffer
 from pypy.module.cpyext.state import State
@@ -265,7 +265,7 @@
     check_num_args(space, w_args, 1)
     args_w = space.fixedview(w_args)
     index = space.int_w(space.index(args_w[0]))
-    null = lltype.nullptr(PyObject.TO)
+    null = rffi.cast(PyObject, 0)
     res = generic_cpy_call(space, func_target, w_self, index, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
@@ -294,7 +294,8 @@
     func_target = rffi.cast(objobjargproc, func)
     check_num_args(space, w_args, 1)
     w_key, = space.fixedview(w_args)
-    res = generic_cpy_call(space, func_target, w_self, w_key, None)
+    null = rffi.cast(PyObject, 0)
+    res = generic_cpy_call(space, func_target, w_self, w_key, null)
     if rffi.cast(lltype.Signed, res) == -1:
         space.fromcache(State).check_and_raise_exception(always=True)
     return space.w_None
@@ -612,6 +613,8 @@
             handled = True
 
     for tp_name, attr in [('tp_hash', '__hash__'),
+                          ('tp_as_sequence.c_sq_length', '__len__'),
+                          ('tp_as_mapping.c_mp_length', '__len__'),
                          ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -637,7 +640,8 @@
                           ('tp_as_number.c_nb_xor', '__xor__'),
                           ('tp_as_number.c_nb_or', '__or__'),
                           ('tp_as_sequence.c_sq_concat', '__add__'),
-                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__')
+                          ('tp_as_sequence.c_sq_inplace_concat', '__iadd__'),
+                          ('tp_as_mapping.c_mp_subscript', '__getitem__'),
                           ]:
         if name == tp_name:
             slot_fn = w_type.getdictvalue(space, attr)
@@ -651,7 +655,7 @@
             handled = True
 
     # binary-with-Py_ssize_t-type
-    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem'),
+    for tp_name, attr in [('tp_as_sequence.c_sq_item', '__getitem__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_repeat', '__mul__'),
                           ('tp_as_sequence.c_sq_inplace_repeat', '__imul__'),
@@ -680,7 +684,48 @@
             def slot_func(space, w_self, w_arg1, w_arg2):
                 return space.call_function(slot_fn, w_self, w_arg1, w_arg2)
             handled = True
+    # ternary-with-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_mapping.c_mp_ass_subscript', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
 
+            @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, w_arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, w_arg1, w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, w_arg1)
+                return 0
+            handled = True
+    # ternary-Py_size_t-void returning-Py_size_t-type
+    for tp_name, attr in [('tp_as_sequence.c_sq_ass_item', '__setitem__'),
+                         ]:
+        if name == tp_name:
+            slot_ass = w_type.getdictvalue(space, attr)
+            if slot_ass is None:
+                return
+            slot_del = w_type.getdictvalue(space, '__delitem__')
+            if slot_del is None:
+                return
+
+            @slot_function([PyObject, lltype.Signed, PyObject], rffi.INT_real, error=-1)
+            @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+            def slot_func(space, w_self, arg1, arg2):
+                if arg2:
+                    w_arg2 = from_ref(space, rffi.cast(PyObject, arg2))
+                    space.call_function(slot_ass, w_self, space.newint(arg1), w_arg2)
+                else:
+                    space.call_function(slot_del, w_self, space.newint(arg1))
+                return 0
+            handled = True
     if handled:
         pass
     elif name == 'tp_setattro':
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -490,29 +490,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyFileObject], lltype.Void)
 def PyFile_IncUseCount(space, p):
     """Increments the PyFileObject's internal use count to indicate
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -1872,6 +1872,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -1887,6 +1888,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -255,4 +255,60 @@
             ])
         d = module.get_type_dict(1)
         assert d['real'].__get__(1, 1) == 1
-
+    def test_advanced(self):
+        module = self.import_extension('foo', [
+            ("dict_len", "METH_O",
+            '''
+                int ret = args->ob_type->tp_as_mapping->mp_length(args);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_setitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 3 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1),
+                        PyTuple_GetItem(args, 2));
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_delitem", "METH_VARARGS",
+            '''
+                int ret;
+                PyObject * dict = PyTuple_GetItem(args, 0);
+                if (PyTuple_Size(args) < 2 || !dict || 
+                        !dict->ob_type->tp_as_mapping ||
+                        !dict->ob_type->tp_as_mapping->mp_ass_subscript)
+                    return PyLong_FromLong(-1);
+                ret = dict->ob_type->tp_as_mapping->mp_ass_subscript(
+                        dict, PyTuple_GetItem(args, 1), NULL);
+                return PyLong_FromLong(ret);
+            '''),
+            ("dict_next", "METH_VARARGS",
+            '''
+                PyObject *key, *value;
+                PyObject *arg = NULL;
+                Py_ssize_t pos = 0;
+                int ret = 0;
+                if ((PyArg_ParseTuple(args, "|O", &arg))) {
+                    if (arg && PyDict_Check(arg)) {
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                        /* test no crash if pos is not reset to 0*/
+                        while (PyDict_Next(arg, &pos, &key, &value))
+                            ret ++;
+                    }
+                }
+                return PyLong_FromLong(ret);
+            '''),
+            ])
+        d = {'a': 1, 'b':2}
+        assert module.dict_len(d) == 2
+        assert module.dict_setitem(d, 'a', 'c') == 0
+        assert d['a'] == 'c'
+        assert module.dict_delitem(d, 'a') == 0
+        r = module.dict_next({'a': 1, 'b': 2})
+        assert r == 2
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -331,3 +331,30 @@
                 def nested_flags():
                     return module.get_flags()""" in ns
         assert ns['nested_flags']() == (1, 0x2000)  # CO_FUTURE_DIVISION
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,11 +38,13 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
-    
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
+
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
@@ -152,29 +154,35 @@
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
-            ("test_macro_invocations", "METH_NOARGS",
+            ("test_macro_invocations", "METH_O",
              """
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
+             PyList_SET_ITEM(o, 0, args);
+             PyList_SET_ITEM(l, 1, args);
 
-             Py_INCREF(o);
-             PyList_SET_ITEM(o, 0, o);
-             Py_INCREF(o);
-             PyList_SET_ITEM(l, 1, o);
+             if(PyList_GET_ITEM(o, 0) != PyList_GET_ITEM(l, 1))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_ITEM error");
+                return NULL;
+             }
 
-             PyList_GET_ITEM(o, 0);
-             PyList_GET_ITEM(l, 1);
-
-             PyList_GET_SIZE(o);
-             PyList_GET_SIZE(l);
+             if(PyList_GET_SIZE(o) != PyList_GET_SIZE(l))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_SIZE error");
+                return NULL;
+             }
 
              return o;
              """
             )
         ])
-        x = module.test_macro_invocations()
-        assert x[0] is x[1] is x
+        s = 'abcdef'
+        x = module.test_macro_invocations(s)
+        assert x[0] is x[1] is s
 
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
@@ -183,39 +191,96 @@
                 PyObject* o, *o2, *o3;
                 o = PyList_New(1);
 
-                o2 = PyInt_FromLong(0);
+                o2 = PyBytes_FromString("test_get_item0");
+                Py_INCREF(o2);
                 PyList_SET_ITEM(o, 0, o2);
-                o2 = NULL;
 
                 o3 = PyList_GET_ITEM(o, 0);
                 Py_INCREF(o3);
-                Py_CLEAR(o);
+                Py_DECREF(o);
+                Py_DECREF(o2);
                 return o3;
              """)])
-        assert module.test_get_item() == 0
+        assert module.test_get_item() == b'test_get_item0'
 
-    def test_set_item_macro(self):
+    def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff_after_setitem", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
-                PyObject* o = PyList_New(0);
-                PyObject* o2 = PyList_New(0);
-                Py_ssize_t refcount, new_refcount;
+                /* test that the refcount differences for functions
+                 * are correct. diff1 - expected refcount diff for i1,
+                 *              diff2 - expected refcount diff for i2
+                 */
+                #define CHECKCOUNT(diff1, diff2, action) \
+                    new_count1 = Py_REFCNT(i1); \
+                    new_count2 = Py_REFCNT(i2); \
+                    diff = new_count1 - old_count1; \
+                    if (diff != diff1) {\
+                        sprintf(errbuffer, action \
+                            " i1 expected diff of %ld got %ld", (long)diff1, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    diff = new_count2 - old_count2; \
+                    if (diff != diff2) {\
+                        sprintf(errbuffer, action \
+                            " i2 expected diff of %ld got %ld", (long)diff2, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    old_count1 = new_count1; \
+                    old_count2 = new_count2;
 
-                PyList_Append(o, o2);  // does not steal o2
+                PyObject* tmp, *o = PyList_New(0);
+                char errbuffer[1024];
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
+                Py_ssize_t old_count1, new_count1;
+                Py_ssize_t old_count2, new_count2;
+                Py_ssize_t diff;
+                int ret;
 
-                refcount = Py_REFCNT(o2);
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
-                // Steal a reference to o2, but leak the old reference to o2.
-                // The net result should be no change in refcount.
-                PyList_SET_ITEM(o, 0, o2);
+                ret = PyList_Append(o, i1);
+                if (ret != 0) 
+                    return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(1, 0, "PyList_Append");
 
-                new_refcount = Py_REFCNT(o2);
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
 
-                Py_CLEAR(o);
-                Py_DECREF(o2); // append incref'd.
-                // Py_CLEAR(o2);  // naive implementation would fail here.
-                return PyLong_FromSsize_t(new_refcount - refcount);
+                PyList_SET_ITEM(o, 0, i2);
+                CHECKCOUNT(0, 0, "PyList_SET_ITEM");
+
+                tmp = PyList_GET_ITEM(o, 0);
+                if (tmp != i2)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(0, 0, "PyList_GET_ITEM");
+
+                PyList_SetItem(o, 0, i1);
+                CHECKCOUNT(0, -1, "PyList_Set_Item");
+
+                PyList_GetItem(o, 0);
+                CHECKCOUNT(0, 0, "PyList_Get_Item");
+
+                Py_DECREF(o); 
+                #ifndef PYPY_VERSION
+                {
+                    // PyPy deletes only at teardown
+                    CHECKCOUNT(-1, 0, "Py_DECREF(o)");
+                }
+                #endif
+                return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff_after_setitem() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -24,7 +24,7 @@
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
+    PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, Py_DecRef, as_pyobj)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -520,7 +520,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.INTP], PyObject)
+ at cpython_api([rffi.CCHARP, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
diff --git a/pypy/module/test_lib_pypy/README.txt b/pypy/module/test_lib_pypy/README.txt
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/README.txt
@@ -0,0 +1,4 @@
+This directory contains app-level tests are supposed to be run *after*
+translation. So you run them by saying:
+
+pypy pytest.py <testfile.py>
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/pypy/module/test_lib_pypy/ctypes_tests/README
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/README
@@ -0,0 +1,8 @@
+-------Ctypes tests------
+
+Unlike the other tests in the PyPy sources, these tests are assumed to run after the translation is complete.
+Therefore, using the resulting binary, you can then call, for example:
+
+/path/to/your_modified_pypy_binary pytest.py pypy/module/test_lib_pypy/ctypes_tests/test_libc.py
+
+This also applies to any other test in ctypes_tests.
\ No newline at end of file
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py b/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from ctypes import CDLL, POINTER, pointer, c_byte, c_int, c_char_p, CFUNCTYPE, c_void_p, c_size_t
-import sys
-import py
-from support import BaseCTypesTestChecker
-
-class MyCDLL(CDLL):
-    def __getattr__(self, attr):
-        fn = self[attr] # this way it's not cached as an attribute
-        fn._slowpath_allowed = False
-        return fn
-
-def setup_module(mod):
-    import conftest
-    _ctypes_test = str(conftest.sofile)
-    mod.dll = MyCDLL(_ctypes_test)  # slowpath not allowed
-    mod.dll2 = CDLL(_ctypes_test)   # slowpath allowed
-
-
-class TestFastpath(BaseCTypesTestChecker):
-
-    def test_fastpath_forbidden(self):
-        def myfunc():
-            pass
-        #
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        #
-        # so far, it's still using the slowpath
-        assert not tf_b._is_fastpath
-        tf_b.callable = myfunc
-        tf_b.argtypes = (c_byte,)
-        # errcheck prevented the fastpath to kick in
-        assert not tf_b._is_fastpath
-        #
-        del tf_b.callable
-        tf_b.argtypes = (c_byte,) # try to re-enable the fastpath
-        assert tf_b._is_fastpath
-        #
-        assert not tf_b._slowpath_allowed
-        py.test.raises(AssertionError, "tf_b.callable = myfunc")
-        py.test.raises(AssertionError, "tf_b('aaa')") # force a TypeError
-
-    def test_simple_args(self):
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        assert tf_b(-126) == -42
-
-    def test_from_cfunctype(self):
-        from _ctypes import _memmove_addr
-        functype = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)
-        my_memmove = functype(_memmove_addr)
-        assert my_memmove._is_fastpath
-
-    def test_undeclared_restype(self):
-        # make sure we get a fresh function
-        try:
-            del dll.tf_i
-        except AttributeError:
-            pass
-        tf_i = dll.tf_i
-        assert not tf_i._is_fastpath
-        tf_i.argtypes = (c_int,)
-        assert tf_i._is_fastpath
-        assert tf_i(12) == 4
-
-    def test_pointer_args(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-        v = c_int(42)
-        result = f(pointer(v))
-        assert type(result) == POINTER(c_int)
-        assert result.contents.value == 42
-
-    def test_simple_pointer_args(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        mystr = c_char_p("abcd")
-        result = f(mystr, ord("b"))
-        assert result == "bcd"
-
-    def test_strings(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        result = f("abcd", ord("b"))
-        assert result == "bcd"
-
-    def test_errcheck(self):
-        def errcheck(result, func, args):
-            return 'hello'
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.errcheck = errcheck
-        assert tf_b(-126) == 'hello'
-
-    def test_array_to_ptr(self):
-        ARRAY = c_int * 8
-        func = dll._testfunc_ai8
-        func.restype = POINTER(c_int)
-        func.argtypes = [ARRAY]
-        array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8)
-        ptr = func(array)
-        assert ptr[0] == 1
-        assert ptr[7] == 8
-
-
-class TestFallbackToSlowpath(BaseCTypesTestChecker):
-
-    def test_argtypes_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_char_p,)  # this is intentionally wrong
-        tf_b.argtypes = None # kill the fast path
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -42
-
-    def test_callable_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.callable = lambda x: x+1
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -125
-        tf_b.callable = None
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -478,7 +478,7 @@
         raises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
 
     def test_argument_conversion_and_checks(self):
-        py.test.skip("XXX currently broken on PyPy, sorry")
+        #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
         strlen = dll.my_strchr
         strlen.argtypes = [c_char_p, c_int]
         strlen.restype = c_char_p
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -611,55 +611,31 @@
         return mod_format(space, w_values, self, do_unicode=False)
 
     def descr_eq(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -677,18 +653,6 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -230,15 +230,13 @@
         return list(items)
 
     def switch_to_object_strategy(self):
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        if self.strategy is object_strategy:
+            return
         list_w = self.getitems()
-        object_strategy = self.space.fromcache(ObjectListStrategy)
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -81,9 +81,6 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
-        if self.config.objspace.std.withstrbuf:
-            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
-
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
@@ -285,7 +282,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/strbufobject.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import SimpleView, StringBuffer
-from pypy.interpreter.error import OperationError
-from rpython.rlib.rstring import StringBuilder
-
-
-class W_StringBufferObject(W_AbstractBytesObject):
-    w_str = None
-
-    def __init__(self, builder):
-        self.builder = builder             # StringBuilder
-        self.length = builder.getlength()
-
-    def force(self):
-        if self.w_str is None:
-            s = self.builder.build()
-            if self.length < len(s):
-                s = s[:self.length]
-            self.w_str = W_BytesObject(s)
-            return s
-        else:
-            return self.w_str._value
-
-    def __repr__(self):
-        """ representation for debugging purposes """
-        return "%s(%r[:%d])" % (
-            self.__class__.__name__, self.builder, self.length)
-
-    def unwrap(self, space):
-        return self.force()
-
-    def str_w(self, space):
-        return self.force()
-
-    def buffer_w(self, space, flags):
-        return SimpleView(StringBuffer(self.force()))
-
-    def descr_len(self, space):
-        return space.newint(self.length)
-
-    def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
-        else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
-
-    def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
-        return self
-
-
-delegation_dict = {}
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
-
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
-
-W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -57,13 +57,6 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
-    def test_withstrbuf_fastpath_isinstance(self):
-        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
-
-        space = gettestobjspace(withstrbuf=True)
-        cls = space._get_interplevel_cls(space.w_bytes)
-        assert cls is W_AbstractBytesObject
-
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import py
-
-from pypy.objspace.std.test import test_bytesobject
-
-class AppTestStringObject(test_bytesobject.AppTestBytesObject):
-    spaceconfig = {"objspace.std.withstrbuf": True}
-
-    def test_basic(self):
-        import __pypy__
-        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
-        # away on AST level
-        s = "Hello, ".__add__("World!")
-        assert type(s) is str
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-
-    def test_add_twice(self):
-        x = "a".__add__("b")
-        y = x + "c"
-        c = x + "d"
-        assert y == "abc"
-        assert c == "abd"
-
-    def test_add(self):
-        import __pypy__
-        all = ""
-        for i in range(20):
-            all += str(i)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
-        assert all == "012345678910111213141516171819"
-
-    def test_hash(self):
-        import __pypy__
-        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
-        t = 'a' * 101
-        s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-        assert hash(s) == hash(t)
-
-    def test_len(self):
-        s = "a".__add__("b")
-        r = "c".__add__("d")
-        t = s + r
-        assert len(s) == 2
-        assert len(r) == 2
-        assert len(t) == 4
-
-    def test_buffer(self):
-        s = b'a'.__add__(b'b')
-        assert buffer(s) == buffer(b'ab')
-        assert memoryview(s) == b'ab'
-
-    def test_add_strbuf(self):
-        # make three strbuf objects
-        s = 'a'.__add__('b')
-        t = 'x'.__add__('c')
-        u = 'y'.__add__('d')
-
-        # add two different strbufs to the same string
-        v = s + t
-        w = s + u
-
-        # check that insanity hasn't resulted.
-        assert v == "abxc"
-        assert w == "abyd"
-
-    def test_more_adding_fun(self):
-        s = 'a'.__add__('b') # s is a strbuf now
-        t = s + 'c'
-        u = s + 'd'
-        v = s + 'e'
-        assert v == 'abe'
-        assert u == 'abd'
-        assert t == 'abc'
-
-    def test_buh_even_more(self):
-        a = 'a'.__add__('b')
-        b = a + 'c'
-        c = '0'.__add__('1')
-        x = c + a
-        assert x == '01ab'
-
-    def test_add_non_string(self):
-        a = 'a'
-        a += 'b'
-        raises(TypeError, "a += 5")

From pypy.commits at gmail.com  Mon May 22 09:32:27 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 22 May 2017 06:32:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix issue #1877: replace bare
 Exceptions with AnnotatorError in the annotator
Message-ID: <5922e86b.ea85df0a.14926.ccda@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91364:1d471fdc1963
Date: 2017-05-22 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1d471fdc1963/

Log:	fix issue #1877: replace bare Exceptions with AnnotatorError in the
	annotator

diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -514,13 +514,13 @@
     ne = eq
 
     def lt((tup1, tup2)):
-        raise Exception("unsupported: (...) < (...)")
+        raise AnnotatorError("unsupported: (...) < (...)")
     def le((tup1, tup2)):
-        raise Exception("unsupported: (...) <= (...)")
+        raise AnnotatorError("unsupported: (...) <= (...)")
     def gt((tup1, tup2)):
-        raise Exception("unsupported: (...) > (...)")
+        raise AnnotatorError("unsupported: (...) > (...)")
     def ge((tup1, tup2)):
-        raise Exception("unsupported: (...) >= (...)")
+        raise AnnotatorError("unsupported: (...) >= (...)")
 
 
 class __extend__(pairtype(SomeDict, SomeDict)):
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -15,7 +15,8 @@
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeException,
     SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked,
-    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty)
+    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty,
+    AnnotatorError)
 from rpython.annotator.classdesc import ClassDef, ClassDesc
 from rpython.annotator.listdef import ListDef, ListItem
 from rpython.annotator.dictdef import DictDef
@@ -343,7 +344,7 @@
         elif x is None:
             return s_None
         else:
-            raise Exception("Don't know how to represent %r" % (x,))
+            raise AnnotatorError("Don't know how to represent %r" % (x,))
         result.const = x
         return result
 
@@ -363,7 +364,7 @@
                 result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
-                    raise Exception("ClassDesc for object not supported")
+                    raise AnnotatorError("ClassDesc for object not supported")
                 if pyobj.__module__ == '__builtin__': # avoid making classdefs for builtin types
                     result = self.getfrozen(pyobj)
                 else:
@@ -400,7 +401,7 @@
                         msg = "object with a __call__ is not RPython"
                     else:
                         msg = "unexpected prebuilt constant"
-                    raise Exception("%s: %r" % (msg, pyobj))
+                    raise AnnotatorError("%s: %r" % (msg, pyobj))
                 result = self.getfrozen(pyobj)
             self.descs[obj_key] = result
             return result
@@ -589,7 +590,7 @@
         for name, value in dict.iteritems():
             if value is func:
                 return cls, name
-    raise Exception("could not match bound-method to attribute name: %r" % (boundmeth,))
+    raise AnnotatorError("could not match bound-method to attribute name: %r" % (boundmeth,))
 
 def ishashable(x):
     try:
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -550,8 +550,8 @@
                                         "with _mixin_: %r" % (cls,))
                 base = b1
         if mixins_before and mixins_after:
-            raise Exception("unsupported: class %r has mixin bases both"
-                            " before and after the regular base" % (self,))
+            raise AnnotatorError("unsupported: class %r has mixin bases both"
+                                 " before and after the regular base" % (self,))
         self.add_mixins(mixins_after, check_not_in=base)
         self.add_mixins(mixins_before)
         self.add_sources_for_class(cls)
@@ -569,8 +569,8 @@
                 attrs.update(decl)
             if self.basedesc is not None:
                 if self.basedesc.all_enforced_attrs is None:
-                    raise Exception("%r has slots or _attrs_, "
-                                    "but not its base class" % (cls,))
+                    raise AnnotatorError("%r has slots or _attrs_, "
+                                         "but not its base class" % (cls,))
                 attrs.update(self.basedesc.all_enforced_attrs)
             self.all_enforced_attrs = attrs
 
@@ -714,8 +714,8 @@
                 try:
                     args.fixedunpack(0)
                 except ValueError:
-                    raise Exception("default __init__ takes no argument"
-                                    " (class %s)" % (self.name,))
+                    raise AnnotatorError("default __init__ takes no argument"
+                                         " (class %s)" % (self.name,))
             elif self.pyobj is Exception:
                 # check explicitly against "raise Exception, x" where x
                 # is a low-level exception pointer
@@ -824,8 +824,8 @@
         # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist,
         # test_immutable_list_out_of_instance.
         if self._detect_invalid_attrs and attr in self._detect_invalid_attrs:
-            raise Exception("field %r was migrated to %r from a subclass in "
-                            "which it was declared as _immutable_fields_" %
+            raise AnnotatorError("field %r was migrated to %r from a subclass in "
+                                 "which it was declared as _immutable_fields_" %
                             (attr, self.pyobj))
         search1 = '%s[*]' % (attr,)
         search2 = '%s?[*]' % (attr,)
@@ -858,7 +858,7 @@
             # call to a single class, look at the result annotation
             # in case it was specialized
             if not isinstance(s_result, SomeInstance):
-                raise Exception("calling a class didn't return an instance??")
+                raise AnnotatorError("calling a class didn't return an instance??")
             classdefs = [s_result.classdef]
         else:
             # call to multiple classes: specialization not supported
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -314,8 +314,8 @@
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
-            raise Exception("%r: signature and enforceargs cannot both be "
-                            "used" % (self,))
+            raise AnnotatorError("%r: signature and enforceargs cannot both be "
+                                 "used" % (self,))
         if enforceargs:
             if not callable(enforceargs):
                 from rpython.annotator.signature import Sig
@@ -432,7 +432,7 @@
     def func_args(self, args):
         from rpython.annotator.model import SomeInstance
         if self.selfclassdef is None:
-            raise Exception("calling %r" % (self,))
+            raise AnnotatorError("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4584,7 +4584,7 @@
             return Ellipsis
         a = self.RPythonAnnotator()
         e = py.test.raises(Exception, a.build_types, f, [])
-        assert str(e.value) == "Don't know how to represent Ellipsis"
+        assert "Don't know how to represent Ellipsis" in str(e.value)
 
     def test_must_be_light_finalizer(self):
         from rpython.rlib import rgc
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -185,8 +185,8 @@
         return SomeString()
 
     def id(self):
-        raise Exception("cannot use id() in RPython; "
-                        "see objectmodel.compute_xxx()")
+        raise AnnotatorError("cannot use id() in RPython; "
+                             "see objectmodel.compute_xxx()")
 
     def int(self):
         return SomeInteger()
@@ -421,7 +421,7 @@
     def setslice(self, s_start, s_stop, s_iterable):
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
-            raise Exception("list[start:stop] = x: x must be a list")
+            raise AnnotatorError("list[start:stop] = x: x must be a list")
         self.listdef.mutate()
         self.listdef.agree(getbookkeeper(), s_iterable.listdef)
         self.listdef.resize()

From pypy.commits at gmail.com  Mon May 22 09:52:11 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Mon, 22 May 2017 06:52:11 -0700 (PDT)
Subject: [pypy-commit] pypy sockopt_zero: proper fix after review
Message-ID: <5922ed0b.0a5e1c0a.55df8.f4a7@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: sockopt_zero
Changeset: r91366:4c2ff9e6a959
Date: 2017-05-22 14:10 +0300
http://bitbucket.org/pypy/pypy/changeset/4c2ff9e6a959/

Log:	proper fix after review

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -305,14 +305,13 @@
         If a nonzero buffersize argument is given, the return value is a
         string of that length; otherwise it is an integer.
         """
-        if w_buflen is None:
+        if w_buflen is not None:
+            buflen = space.int_w(w_buflen)
+        if w_buflen is None or buflen == 0:
             try:
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
-        buflen = space.int_w(w_buflen)
-        if buflen == 0:
-            return space.newint(0)
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -582,15 +582,18 @@
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
 
-    def test_getsockopt_zero(self):
-        # related to issue #2561: in CPython, when setting the buffer size:
-        # if 0, should return 0,
+    def test_getsetsockopt_zero(self):
+        # related to issue #2561: when specifying the buffer size param:
+        # if 0 or None, should return the setted value,
         # otherwise an empty buffer of the specified size
         import _socket
         s = _socket.socket()
-
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
 
     def test_socket_ioctl(self):
         import _socket, sys

From pypy.commits at gmail.com  Mon May 22 09:52:14 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 22 May 2017 06:52:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 palecsandru/pypy2_sockopt_zero/sockopt_zero (pull request #548)
Message-ID: <5922ed0e.129ddf0a.665a1.ce5a@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91368:965ffd1ed1de
Date: 2017-05-22 13:51 +0000
http://bitbucket.org/pypy/pypy/changeset/965ffd1ed1de/

Log:	Merged in palecsandru/pypy2_sockopt_zero/sockopt_zero (pull request
	#548)

	Passing a buffersize of 0 to socket.getsockopt

	Approved-by: Carl Friedrich Bolz <cfbolz at gmx.de>

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -297,20 +297,19 @@
         except SocketError as e:
             raise converted_error(space, e)
 
-    @unwrap_spec(level=int, optname=int)
-    def getsockopt_w(self, space, level, optname, w_buflen=None):
+    @unwrap_spec(level=int, optname=int, buflen=int)
+    def getsockopt_w(self, space, level, optname, buflen=0):
         """getsockopt(level, option[, buffersize]) -> value
 
         Get a socket option.  See the Unix manual for level and option.
         If a nonzero buffersize argument is given, the return value is a
         string of that length; otherwise it is an integer.
         """
-        if w_buflen is None:
+        if buflen == 0:
             try:
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
-        buflen = space.int_w(w_buflen)
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -582,6 +582,19 @@
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
 
+    def test_getsetsockopt_zero(self):
+        # related to issue #2561: when specifying the buffer size param:
+        # if 0 or None, should return the setted value,
+        # otherwise an empty buffer of the specified size
+        import _socket
+        s = _socket.socket()
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':

From pypy.commits at gmail.com  Mon May 22 09:52:08 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Mon, 22 May 2017 06:52:08 -0700 (PDT)
Subject: [pypy-commit] pypy sockopt_zero: added the possibility to have zero
 sized buffer length
Message-ID: <5922ed08.ca98df0a.4cfa0.6536@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: sockopt_zero
Changeset: r91365:74c6874f803f
Date: 2017-05-22 11:02 +0300
http://bitbucket.org/pypy/pypy/changeset/74c6874f803f/

Log:	added the possibility to have zero sized buffer length

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -311,6 +311,8 @@
             except SocketError as e:
                 raise converted_error(space, e)
         buflen = space.int_w(w_buflen)
+        if buflen == 0:
+            return space.newint(0)
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -582,6 +582,16 @@
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
 
+    def test_getsockopt_zero(self):
+        # related to issue #2561: in CPython, when setting the buffer size:
+        # if 0, should return 0,
+        # otherwise an empty buffer of the specified size
+        import _socket
+        s = _socket.socket()
+
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':

From pypy.commits at gmail.com  Mon May 22 09:52:13 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Mon, 22 May 2017 06:52:13 -0700 (PDT)
Subject: [pypy-commit] pypy sockopt_zero: shorten code path
Message-ID: <5922ed0d.505c1c0a.1a1ed.3942@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: sockopt_zero
Changeset: r91367:f848cd62800a
Date: 2017-05-22 16:31 +0300
http://bitbucket.org/pypy/pypy/changeset/f848cd62800a/

Log:	shorten code path

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -297,17 +297,15 @@
         except SocketError as e:
             raise converted_error(space, e)
 
-    @unwrap_spec(level=int, optname=int)
-    def getsockopt_w(self, space, level, optname, w_buflen=None):
+    @unwrap_spec(level=int, optname=int, buflen=int)
+    def getsockopt_w(self, space, level, optname, buflen=0):
         """getsockopt(level, option[, buffersize]) -> value
 
         Get a socket option.  See the Unix manual for level and option.
         If a nonzero buffersize argument is given, the return value is a
         string of that length; otherwise it is an integer.
         """
-        if w_buflen is not None:
-            buflen = space.int_w(w_buflen)
-        if w_buflen is None or buflen == 0:
+        if buflen == 0:
             try:
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:

From pypy.commits at gmail.com  Mon May 22 09:59:27 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 22 May 2017 06:59:27 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5922eebf.0795df0a.54acb.eaf0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91369:619bc1ea9e6d
Date: 2017-05-22 14:58 +0100
http://bitbucket.org/pypy/pypy/changeset/619bc1ea9e6d/

Log:	fix test

diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.pyobject import from_ref
 
 class TestListObject(BaseApiTest):
     def test_list(self, space, api):
@@ -38,12 +39,10 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
-                                    space.wrap(2)) == 0
+        assert space.unwrap(from_ref(space, api.PyList_GetItem(w_l, 3))) == 2
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
-                                    space.wrap(3)) == 0
+        assert space.unwrap(from_ref(space, api.PyList_GET_ITEM(w_l, 3))) == 3
 
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
@@ -243,7 +242,7 @@
                 old_count2 = Py_REFCNT(i2);
 
                 ret = PyList_Append(o, i1);
-                if (ret != 0) 
+                if (ret != 0)
                     return NULL;
                 /* check the result of Append(), and also force the list
                    to use the CPyListStrategy now */
@@ -274,7 +273,7 @@
                 PyList_GetItem(o, 0);
                 CHECKCOUNT(0, 0, "PyList_Get_Item");
 
-                Py_DECREF(o); 
+                Py_DECREF(o);
                 #ifndef PYPY_VERSION
                 {
                     // PyPy deletes only at teardown

From pypy.commits at gmail.com  Mon May 22 10:17:38 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 22 May 2017 07:17:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Adapt test to py3
Message-ID: <5922f302.daa2df0a.d4f1c.15cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91370:e8a4ea639258
Date: 2017-05-22 15:17 +0100
http://bitbucket.org/pypy/pypy/changeset/e8a4ea639258/

Log:	Adapt test to py3

diff --git a/pypy/module/cpyext/test/issue2482.c b/pypy/module/cpyext/test/issue2482.c
--- a/pypy/module/cpyext/test/issue2482.c
+++ b/pypy/module/cpyext/test/issue2482.c
@@ -99,9 +99,12 @@
     Py_INCREF(base);
     type->tp_base = (PyTypeObject *) base;
     type->tp_basicsize = ((PyTypeObject *) base)->tp_basicsize;
-    type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_CHECKTYPES;
+    type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
 
     if (PyType_Ready(type) < 0) INITERROR;
 
     PyModule_AddObject(module, name, (PyObject *) type);
+#if PY_MAJOR_VERSION >= 3
+    return module;
+#endif
 };

From pypy.commits at gmail.com  Mon May 22 11:35:13 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 May 2017 08:35:13 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: add passing tests
Message-ID: <59230531.09631c0a.443ae.267b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91371:21ce0ac7d0de
Date: 2017-05-22 18:34 +0300
http://bitbucket.org/pypy/pypy/changeset/21ce0ac7d0de/

Log:	add passing tests

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -143,6 +143,13 @@
         assert fuu2(u"abc").baz().escape()
         raises(TypeError, module.fooType.object_member.__get__, 1)
 
+    def test_hash_subtype(self):
+        module = self.import_module(name='foo')
+        newobj = module.UnicodeSubtype(u"xyz")
+        assert hash(newobj) == hash(u"xyz")
+        assert hash(module.UnicodeSubtype(u"")) in [0, -2]
+        
+
     def test_multiple_inheritance1(self):
         module = self.import_module(name='foo')
         obj = module.UnicodeSubtype(u'xyz')
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -741,7 +741,6 @@
     def test_hash(self):
         # check that we have the same hash as CPython for at least 31 bits
         # (but don't go checking CPython's special case -1)
-        # disabled: assert hash('') == 0 --- different special case
         assert hash('hello') & 0x7fffffff == 0x347697fd
         assert hash('hello world!') & 0x7fffffff == 0x2f0bb411
         assert hash('') in [0, -2]
diff --git a/pypy/objspace/std/test/test_unicodeobject.py b/pypy/objspace/std/test/test_unicodeobject.py
--- a/pypy/objspace/std/test/test_unicodeobject.py
+++ b/pypy/objspace/std/test/test_unicodeobject.py
@@ -1012,3 +1012,7 @@
         raises(TypeError, "unicode('', encoding=None)")
         raises(TypeError, 'u"".encode("utf-8", None)')
 
+    def test_hash(self):
+        assert hash(u'hello') & 0x7fffffff == 0x347697fd
+        assert hash(u'hello world!') & 0x7fffffff == 0x2f0bb411
+        assert hash(u'') in [0, -2]

From pypy.commits at gmail.com  Mon May 22 11:58:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 22 May 2017 08:58:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add validation of
 __getnewargs__()/__getnewargs_ex__() return values
Message-ID: <59230abd.6590500a.5e15a.8ba2@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91372:48119cb4c0db
Date: 2017-05-22 16:58 +0100
http://bitbucket.org/pypy/pypy/changeset/48119cb4c0db/

Log:	Add validation of __getnewargs__()/__getnewargs_ex__() return values

diff --git a/pypy/objspace/std/objectobject.py b/pypy/objspace/std/objectobject.py
--- a/pypy/objspace/std/objectobject.py
+++ b/pypy/objspace/std/objectobject.py
@@ -173,31 +173,55 @@
     return space.get_and_call_function(w_impl, w_obj)
 
 
+def _getnewargs(space, w_obj):
+    w_descr = space.lookup(w_obj, '__getnewargs_ex__')
+    hasargs = True
+    if w_descr is not None:
+        w_result = space.get_and_call_function(w_descr, w_obj)
+        if not space.isinstance_w(w_result, space.w_tuple):
+            raise oefmt(space.w_TypeError,
+                "__getnewargs_ex__ should return a tuple, not '%T'", w_result)
+        n = space.len_w(w_result)
+        if n != 2:
+            raise oefmt(space.w_ValueError,
+                "__getnewargs_ex__ should return a tuple of length 2, not %d",
+                n)
+        w_args, w_kwargs = space.fixedview(w_result, 2)
+        if not space.isinstance_w(w_args, space.w_tuple):
+            raise oefmt(space.w_TypeError,
+                "first item of the tuple returned by __getnewargs_ex__ must "
+                "be a tuple, not '%T'", w_args)
+        if not space.isinstance_w(w_kwargs, space.w_dict):
+            raise oefmt(space.w_TypeError,
+                "second item of the tuple returned by __getnewargs_ex__ must "
+                "be a dict, not '%T'", w_kwargs)
+    else:
+        w_descr = space.lookup(w_obj, '__getnewargs__')
+        if w_descr is not None:
+            w_args = space.get_and_call_function(w_descr, w_obj)
+            if not space.isinstance_w(w_args, space.w_tuple):
+                raise oefmt(space.w_TypeError,
+                    "__getnewargs__ should return a tuple, not '%T'", w_args)
+        else:
+            hasargs = False
+            w_args = space.newtuple([])
+        w_kwargs = space.w_None
+    return hasargs, w_args, w_kwargs
+
 @unwrap_spec(proto=int)
 def descr__reduce__(space, w_obj, proto=0):
     w_proto = space.newint(proto)
     if proto >= 2:
-        w_descr = space.lookup(w_obj, '__getnewargs_ex__')
-        hasargs = True
-        if w_descr is not None:
-            w_result = space.get_and_call_function(w_descr, w_obj)
-            w_args, w_kwargs = space.fixedview(w_result, 2)
-        else:
-            w_descr = space.lookup(w_obj, '__getnewargs__')
-            if w_descr is not None:
-                w_args = space.get_and_call_function(w_descr, w_obj)
-            else:
-                hasargs = False
-                w_args = space.newtuple([])
-            w_kwargs = space.w_None
+        hasargs, w_args, w_kwargs = _getnewargs(space, w_obj)
         w_getstate = space.lookup(w_obj, '__get_state__')
         if w_getstate is None:
-            required = not hasargs and \
-                       not space.isinstance_w(w_obj, space.w_list) and \
-                       not space.isinstance_w(w_obj, space.w_dict)
+            required = (not hasargs and
+                not space.isinstance_w(w_obj, space.w_list) and
+                not space.isinstance_w(w_obj, space.w_dict))
             w_obj_type = space.type(w_obj)
             if required and w_obj_type.layout.typedef.variable_sized:
-                raise oefmt(space.w_TypeError, "cannot pickle %N objects", w_obj_type)
+                raise oefmt(
+                    space.w_TypeError, "cannot pickle %N objects", w_obj_type)
         return reduce_2(space, w_obj, w_proto, w_args, w_kwargs)
     return reduce_1(space, w_obj, w_proto)
 
diff --git a/pypy/objspace/std/test/test_obj.py b/pypy/objspace/std/test/test_obj.py
--- a/pypy/objspace/std/test/test_obj.py
+++ b/pypy/objspace/std/test/test_obj.py
@@ -89,6 +89,47 @@
         assert '__getnewargs__' not in seen
         assert '__getnewargs_ex__' not in seen
 
+    def test_reduce_ex_errors(self):
+        # cf. lib-python/3/test/test_descr.py::PicklingTests.test_reduce()
+        args = (-101, "spam")
+        kwargs = {'bacon': -201, 'fish': -301}
+
+        class C2:
+            def __getnewargs__(self):
+                return "bad args"
+        excinfo = raises(TypeError, C2().__reduce_ex__, 4)
+        assert str(excinfo.value) == \
+            "__getnewargs__ should return a tuple, not 'str'"
+
+        class C4:
+            def __getnewargs_ex__(self):
+                return (args, "bad dict")
+        excinfo = raises(TypeError, C4().__reduce_ex__, 4)
+        assert str(excinfo.value) == ("second item of the tuple "
+            "returned by __getnewargs_ex__ must be a dict, not 'str'")
+
+        class C5:
+            def __getnewargs_ex__(self):
+                return ("bad tuple", kwargs)
+        excinfo = raises(TypeError, C5().__reduce_ex__, 4)
+        assert str(excinfo.value) == ("first item of the tuple "
+            "returned by __getnewargs_ex__ must be a tuple, not 'str'")
+
+        class C6:
+            def __getnewargs_ex__(self):
+                return ()
+        excinfo = raises(ValueError, C6().__reduce_ex__, 4)
+        assert str(excinfo.value) == \
+            "__getnewargs_ex__ should return a tuple of length 2, not 0"
+
+        class C7:
+            def __getnewargs_ex__(self):
+                return "bad args"
+        excinfo = raises(TypeError, C7().__reduce_ex__, 4)
+        assert str(excinfo.value) == \
+            "__getnewargs_ex__ should return a tuple, not 'str'"
+
+
     def test_reduce_state_empty_dict(self):
         class X(object):
             pass

From pypy.commits at gmail.com  Mon May 22 12:32:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 22 May 2017 09:32:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backout PR #490 (ab8a0b092d2d +
 follow-up bd7b1902c4df).
Message-ID: <592312ab.493e1c0a.eea48.8bff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91373:1a06512dea4b
Date: 2017-05-22 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/1a06512dea4b/

Log:	Backout PR #490 (ab8a0b092d2d + follow-up bd7b1902c4df).

	See https://bitbucket.org/pypy/pypy/issues/2534. It gives a smallish
	PyPy example which crashes because of this PR #490. For more infos
	see https://bitbucket.org/pypy/pypy/pull-requests/490/.

diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -110,7 +110,7 @@
     """
     log_noopt = False
 
-    def __init__(self, trace, celltoken, state, runtime_boxes,
+    def __init__(self, trace, celltoken, state,
                  call_pure_results=None, enable_opts=None,
                  inline_short_preamble=True):
         self.trace = trace
@@ -119,8 +119,6 @@
         self.state = state
         self.call_pure_results = call_pure_results
         self.inline_short_preamble = inline_short_preamble
-        assert runtime_boxes is not None
-        self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -128,11 +126,7 @@
         assert unroll # we should not be here if it's disabled
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.runtime_boxes, self.call_pure_results, self.inline_short_preamble)
-
-    def forget_optimization_info(self):
-        self.state.forget_optimization_info()
-        CompileData.forget_optimization_info(self)
+            self.call_pure_results, self.inline_short_preamble)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
@@ -298,7 +292,7 @@
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     jitcell_token.target_tokens = [start_descr]
-    loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
+    loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -368,7 +362,7 @@
     history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
     enable_opts = jitdriver_sd.warmstate.enable_opts
     call_pure_results = metainterp.call_pure_results
-    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, jumpargs,
+    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -380,7 +374,6 @@
         history.cut(cut)
         history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
         loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
-                                     jumpargs,
                                      call_pure_results=call_pure_results,
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
@@ -525,7 +518,7 @@
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
-        #     somehow for ContinueRunningNormally
+        #     somehoe for ContinueRunningNormally
         if reset_values:
             item.reset_value()
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
      AbstractResOp, GuardResOp
-from rpython.rlib.objectmodel import we_are_translated, we_are_debug
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.optimizeopt import info
         
 
@@ -173,7 +173,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getfield(descr, optheap)
-        if we_are_debug() and res:
+        if not we_are_translated() and res:
             if isinstance(opinfo, info.AbstractStructPtrInfo):
                 assert opinfo in self.cached_infos
         if isinstance(res, PreambleOp):
@@ -203,7 +203,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getitem(descr, self.index, optheap)
-        if we_are_debug() and res:
+        if not we_are_translated() and res:
             if isinstance(opinfo, info.ArrayPtrInfo):
                 assert opinfo in self.cached_infos
         if (isinstance(res, PreambleOp) and
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,20 +24,9 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
-def check_no_forwarding(lsts):
-    for lst in lsts:
-        for op in lst:
-            assert op.get_forwarded() is None
-
 class LoopInfo(object):
     label_op = None
 
-    def _check_no_forwarding(self):
-        pass
-
-    def forget_optimization_info(self):
-        pass
-
 class BasicLoopInfo(LoopInfo):
     def __init__(self, inputargs, quasi_immutable_deps, jump_op):
         self.inputargs = inputargs
@@ -567,8 +556,7 @@
         return (BasicLoopInfo(trace.inputargs, self.quasi_immutable_deps, last_op),
                 self._newoperations)
 
-    @staticmethod
-    def _clean_optimization_info(lst):
+    def _clean_optimization_info(self, lst):
         for op in lst:
             if op.get_forwarded() is not None:
                 op.set_forwarded(None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -5,7 +5,6 @@
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
-from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 
 class PreambleOp(AbstractResOp):
@@ -19,7 +18,7 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     op = None
-
+    
     def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
@@ -52,13 +51,7 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
-    res = None
-
-    def _check_no_forwarding(self):
-        assert self.res.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        self.res.clear_forwarded()
+    pass
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -108,14 +101,6 @@
                                        descr=sop.getdescr())
         return ProducedShortOp(self, preamble_op)
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        assert self.getfield_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.getfield_op.clear_forwarded()
-
     def __repr__(self):
         return "HeapOp(%r)" % (self.res,)
 
@@ -208,16 +193,6 @@
                 l.append(pop)
         return l
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        self.one._check_no_forwarding()
-        self.two._check_no_forwarding()
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.one.forget_optimization_info()
-        self.two.forget_optimization_info()
-
     def repr(self, memo):
         return "CompoundOp(%s, %s, %s)" % (self.res.repr(memo),
                                            self.one.repr(memo),
@@ -228,7 +203,7 @@
 
 class ProducedShortOp(AbstractProducedShortOp):
     invented_name = False
-
+    
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
@@ -240,14 +215,6 @@
     def repr(self, memo):
         return self.short_op.repr(memo)
 
-    def _check_no_forwarding(self):
-        self.short_op._check_no_forwarding()
-        assert self.preamble_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        self.short_op.forget_optimization_info()
-        self.preamble_op.clear_forwarded()
-
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
 
@@ -268,14 +235,6 @@
     def repr(self, memo):
         return "INP(%s)" % (self.res.repr(memo),)
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        assert self.preamble_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.preamble_op.clear_forwarded()
-
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
@@ -495,23 +454,16 @@
         self.sb = sb
         self.extra_same_as = self.sb.extra_same_as
         self.target_token = target_token
-        self.build_inplace = False
 
     def setup(self, jump_args, short, label_args):
         self.jump_args = jump_args
         self.short = short
         self.label_args = label_args
-        self.build_inplace = True
 
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
         """
-        # Could this be considered a speculative error?
-        # This check should only fail when trying to jump to an existing trace
-        # by forcing portions of the virtualstate.
-        if not self.build_inplace:
-            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         op = preamble_op.op.get_box_replacement()
         if preamble_op.invented_name:
             self.extra_same_as.append(op)
@@ -519,8 +471,6 @@
         self.jump_args.append(preamble_op.preamble_op)
 
     def use_box(self, box, preamble_op, optimizer=None):
-        if not self.build_inplace:
-            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         jump_op = self.short.pop()
         AbstractShortPreambleBuilder.use_box(self, box, preamble_op, optimizer)
         self.short.append(jump_op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -577,7 +577,6 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
-        state[0]._check_no_forwarding()
         return state
 
     def _convert_call_pure_results(self, d):
@@ -626,7 +625,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
         loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, runtime_boxes, call_pure_results)
+            celltoken, start_state, call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimize import InvalidLoop, SpeculativeError
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo, check_no_forwarding
+     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
@@ -35,7 +35,7 @@
 
     def setinfo_from_preamble_list(self, lst, infos):
         for item in lst:
-            if item is None or isinstance(item, Const):
+            if item is None:
                 continue
             i = infos.get(item, None)
             if i is not None:
@@ -99,6 +99,7 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
+
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
@@ -116,22 +117,26 @@
         return modifier.get_virtual_state(args)
 
     def _check_no_forwarding(self, lsts, check_newops=True):
-        check_no_forwarding(lsts)
+        for lst in lsts:
+            for op in lst:
+                assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
 
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, memo)
+                                           info.inputargs,
+                                           runtime_boxes, memo)
         exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
+    def optimize_peeled_loop(self, trace, celltoken, state,
                              call_pure_results, inline_short_preamble=True):
         trace = trace.get_iter()
         try:
@@ -183,7 +188,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, runtime_boxes, force_boxes=False)
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -196,7 +201,7 @@
             # to the preamble.
             try:
                 new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, runtime_boxes, force_boxes=True)
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
@@ -279,7 +284,8 @@
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, box_names_memo)
+                                           info.inputargs, runtime_boxes,
+                                           box_names_memo)
         exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
@@ -443,7 +449,8 @@
                 continue
             self._expand_info(item, infos)
 
-    def export_state(self, original_label_args, renamed_inputargs, memo):
+    def export_state(self, original_label_args, renamed_inputargs,
+                     runtime_boxes, memo):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
         self.optimizer.flush()
@@ -464,17 +471,16 @@
             op = produced_op.short_op.res
             if not isinstance(op, Const):
                 self._expand_info(op, infos)
+        self.optimizer._clean_optimization_info(end_args)
         return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
-                             short_inputargs, memo)
+                             short_inputargs, runtime_boxes, memo)
 
     def import_state(self, targetargs, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
                 len(targetargs))
-        self._check_no_forwarding([targetargs])
-        exported_state._check_no_forwarding()
         for i, target in enumerate(exported_state.next_iteration_args):
             source = targetargs[i]
             assert source is not target
@@ -530,11 +536,13 @@
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
     * quasi_immutable_deps - for tracking quasi immutables
+    * runtime_boxes - runtime values for boxes, necessary when generating
+                      guards to jump to
     """
 
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
-                 short_inputargs, memo):
+                 short_inputargs, runtime_boxes, memo):
         self.end_args = end_args
         self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
@@ -542,8 +550,8 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
+        self.runtime_boxes = runtime_boxes
         self.dump(memo)
-        self.forget_optimization_info()
 
     def dump(self, memo):
         if have_debug_prints():
@@ -553,35 +561,5 @@
                 debug_print("  " + box.repr(memo))
             debug_stop("jit-log-exported-state")
 
-    def _check_no_forwarding(self):
-        """ Ensures that no optimization state is attached to relevant operations
-        before importing anything. """
-        # Some of these may be redunant
-        check_no_forwarding([
-            self.end_args,
-            self.next_iteration_args,
-            self.renamed_inputargs,
-            self.short_inputargs,
-            self.exported_infos.keys()])
-        for box in self.short_boxes:
-            box._check_no_forwarding()
-
-    def forget_optimization_info(self):
-        """ Clean up optimization info on all operations stored in the ExportedState.
-
-        This function needs to be called when exporting the optimizer state to
-        prevent leaking of optimization information between invocations of the
-        optimizer.
-
-        That includes cleaning up in the event that optimize_peeled_loop() fails
-        with an InvalidLoop exception, as optimize_peeled_loop() mutates the
-        contents of ExportedState.
-        """
-        Optimizer._clean_optimization_info(self.renamed_inputargs)
-        for box in self.exported_infos.iterkeys():
-            box.clear_forwarded()
-        for box in self.short_boxes:
-            box.forget_optimization_info()
-
     def final(self):
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -2024,8 +2024,6 @@
         self.aborted_tracing_greenkey = None
 
     def retrace_needed(self, trace, exported_state):
-        if not we_are_translated():
-            exported_state._check_no_forwarding()
         self.partial_trace = trace
         self.retracing_from = self.potential_retrace_position
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -52,10 +52,6 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
-    def clear_forwarded(self):
-        if self.get_forwarded() is not None:
-            self.set_forwarded(None)
-
     @specialize.arg(1)
     def get_box_replacement(op, not_const=False):
         # Read the chain "op, op._forwarded, op._forwarded._forwarded..."
diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -295,15 +295,10 @@
 
 malloc_zero_filled = CDefinedIntSymbolic('MALLOC_ZERO_FILLED', default=0)
 _translated_to_c = CDefinedIntSymbolic('1 /*_translated_to_c*/', default=0)
-_rpy_assert_value = CDefinedIntSymbolic('RPY_ASSERT_VALUE', default=1)
 
 def we_are_translated_to_c():
     return we_are_translated() and _translated_to_c
 
-def we_are_debug():
-    """ Returns True when not translated or translated with debugging enabled. """
-    return not we_are_translated() or (_translated_to_c and _rpy_assert_value)
-
 # ____________________________________________________________
 
 def instantiate(cls, nonmovable=False):
diff --git a/rpython/translator/c/src/support.h b/rpython/translator/c/src/support.h
--- a/rpython/translator/c/src/support.h
+++ b/rpython/translator/c/src/support.h
@@ -31,10 +31,8 @@
 RPY_EXTERN
 void RPyAssertFailed(const char* filename, long lineno,
                      const char* function, const char *msg);
-#  define RPY_ASSERT_VALUE 1
 #else
 #  define RPyAssert(x, msg)   /* nothing */
-#  define RPY_ASSERT_VALUE 0
 #endif
 
 RPY_EXTERN

From pypy.commits at gmail.com  Mon May 22 13:03:49 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 10:03:49 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: backout 85d3ab6fe80b,
 which was broken, and start to rewrite the logic in a more generic way. Also,
 write tests in such a way that we can check that gc_{load,
 store}_indexed produce effects corresponding to their equivalent
Message-ID: <592319f5.c7a1df0a.1b43f.dcd0@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91374:356dacc477eb
Date: 2017-05-22 18:30 +0200
http://bitbucket.org/pypy/pypy/changeset/356dacc477eb/

Log:	backout 85d3ab6fe80b, which was broken, and start to rewrite the
	logic in a more generic way. Also, write tests in such a way that we
	can check that gc_{load,store}_indexed produce effects corresponding
	to their equivalent

diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -1,5 +1,7 @@
 import py
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.translator.backendopt.writeanalyze import WriteAnalyzer, top_set
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -269,34 +271,6 @@
         assert struct == "struct"
         assert name.endswith("foobar")
 
-    def test_gc_store_indexed(self):
-        from rpython.rtyper.lltypesystem import lltype, llmemory
-        from rpython.rtyper.lltypesystem.lloperation import llop
-        from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-                                      ll_for_resizable_list)
-
-        def write_item(lst, byte_offset, value):
-            ll_data = ll_for_resizable_list(lst)
-            ll_items = ll_data.items
-            LIST = lltype.typeOf(ll_data).TO # rlist.LIST_OF(lltype.Char)
-            base_ofs = llmemory.itemoffsetof(LIST.items.TO, 0)
-            scale_factor = llmemory.sizeof(lltype.Char)
-            llop.gc_store_indexed(lltype.Void, ll_items, byte_offset, value,
-                                  scale_factor, base_ofs)
-
-        def f(x):
-            lst = resizable_list_supporting_raw_ptr(['A', 'B', 'C', 'D'])
-            write_item(lst, 0, 'E')
-        t, wa = self.translate(f, [int])
-        fgraph = graphof(t, f)
-        result = wa.analyze(fgraph.startblock.operations[-1])
-        #
-        assert len(result) == 1
-        array, A = list(result)[0]
-        assert array == "array"
-        assert A.TO.OF == lltype.Char
-
-
 
 class TestLLtypeReadWriteAnalyze(BaseTest):
     Analyzer = ReadWriteAnalyzer
@@ -433,3 +407,42 @@
         res = list(result)
         assert ('readinteriorfield', lltype.Ptr(A), 'y') in res
         assert ('interiorfield', lltype.Ptr(A), 'x') in res
+
+
+class TestGcLoadStoreIndexed(BaseTest):
+    Analyzer = ReadWriteAnalyzer
+
+    def _analyze_graph_single(self, t, wa, fn):
+        graph = graphof(t, fn)
+        result = wa.analyze(graph.startblock.operations[-1])
+        result = list(result)
+        return result
+
+    def test_gc_load_indexed_str(self):
+        from rpython.rlib.buffer import StringBuffer
+
+        def typed_read(buf):
+            return buf.typed_read(lltype.Signed, 0)
+
+        def direct_read(buf):
+            return buf.value[0]
+
+        def f(x):
+            buf = StringBuffer(x)
+            return direct_read(buf), typed_read(buf)
+
+        t, wa = self.translate(f, [str])
+        # check that the effect of direct_read
+        direct_effects = self._analyze_graph_single(t, wa, direct_read)
+        assert len(direct_effects) == 1
+        effect = direct_effects[0]
+        what, T, field = effect
+        assert what == 'readinteriorfield'
+        assert T == lltype.Ptr(STR)
+        assert field == 'chars'
+        #
+        # typed_read contains many effects because it reads the vtable etc.,
+        # but we want to check that it contains also the same effect as
+        # direct_read
+        typed_effects = self._analyze_graph_single(t, wa, typed_read)
+        assert effect in typed_effects
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -1,5 +1,6 @@
 from rpython.flowspace.model import Variable, Constant
 from rpython.translator.backendopt import graphanalyze
+from rpython.rtyper.lltypesystem import lltype, llmemory
 
 top_set = object()
 empty_set = frozenset()
@@ -62,8 +63,7 @@
                 name = self._getinteriorname(op)
                 return self._interiorfield_result(op.args[0].concretetype, name)
         elif op.opname == "gc_store_indexed":
-            if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]):
-                return self._array_result(op.args[0].concretetype)
+            assert False, 'implement me'
         return empty_set
 
     def _array_result(self, TYPE):
@@ -125,4 +125,37 @@
             name = self._getinteriorname(op)
             return frozenset([("readinteriorfield", op.args[0].concretetype,
                             name)])
+        elif op.opname == "gc_load_indexed":
+            return self._gc_load_store_result(op)
         return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo)
+
+    def _gc_load_store_result(self, op):
+        base_offset = op.args[3].value
+        effect = self._get_effect_for_offset(base_offset)
+        return frozenset([effect])
+
+    def _get_effect_for_offset(self, ofs):
+        # gc_{load,store}_indexed are generic operation which operate on
+        # various data types: depending on the symbolic offset, they can be
+        # equivalent as reading/writing an array, and interiorfield, etc. The
+        # following logic tries to catch all the known usages. If you see an
+        # 'implement me', please update the logic accordingly
+        if isinstance(ofs, llmemory.CompositeOffset):
+            # get the effect for the first component and modify it if
+            # necessary
+            sub_offsets = ofs.offsets
+            effect = self._get_effect_for_offset(sub_offsets[0])
+            for sub_ofs in sub_offsets[1:]:
+                if isinstance(sub_ofs, llmemory.ArrayItemsOffset):
+                    # reading from the middle of an array is the same as
+                    # reading from the beginning, so we don't need to change
+                    # the effect
+                    pass
+                else:
+                    assert False, 'implement me'
+            return effect
+        elif isinstance(ofs, llmemory.FieldOffset):
+            T = ofs.TYPE
+            return ('readinteriorfield', lltype.Ptr(T), ofs.fldname)
+        else:
+            assert False, 'implement me'

From pypy.commits at gmail.com  Mon May 22 13:03:51 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 10:03:51 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add support & test for
 gc_load_indexed of a list of chars
Message-ID: <592319f7.442f1c0a.ada97.6460@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91375:d5be1a60c5a3
Date: 2017-05-22 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/d5be1a60c5a3/

Log:	add support & test for gc_load_indexed of a list of chars

diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype, llmemory
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rlist import LIST_OF
 from rpython.translator.translator import TranslationContext, graphof
 from rpython.translator.backendopt.writeanalyze import WriteAnalyzer, top_set
 from rpython.translator.backendopt.writeanalyze import ReadWriteAnalyzer
@@ -412,10 +413,9 @@
 class TestGcLoadStoreIndexed(BaseTest):
     Analyzer = ReadWriteAnalyzer
 
-    def _analyze_graph_single(self, t, wa, fn):
+    def _analyze_graph(self, t, wa, fn):
         graph = graphof(t, fn)
         result = wa.analyze(graph.startblock.operations[-1])
-        result = list(result)
         return result
 
     def test_gc_load_indexed_str(self):
@@ -433,16 +433,46 @@
 
         t, wa = self.translate(f, [str])
         # check that the effect of direct_read
-        direct_effects = self._analyze_graph_single(t, wa, direct_read)
-        assert len(direct_effects) == 1
-        effect = direct_effects[0]
-        what, T, field = effect
-        assert what == 'readinteriorfield'
-        assert T == lltype.Ptr(STR)
-        assert field == 'chars'
+        direct_effects = self._analyze_graph(t, wa, direct_read)
+        assert direct_effects == frozenset([
+            ('readinteriorfield', lltype.Ptr(STR), 'chars')
+        ])
         #
         # typed_read contains many effects because it reads the vtable etc.,
         # but we want to check that it contains also the same effect as
         # direct_read
-        typed_effects = self._analyze_graph_single(t, wa, typed_read)
-        assert effect in typed_effects
+        typed_effects = self._analyze_graph(t, wa, typed_read)
+        assert direct_effects.issubset(typed_effects)
+
+    def test_gc_load_indexed_list_of_chars(self):
+        from rpython.rlib.buffer import ByteBuffer
+
+        def typed_read(buf):
+            return buf.typed_read(lltype.Signed, 0)
+
+        def direct_read(buf):
+            return buf.data[0]
+
+        def f(x):
+            buf = ByteBuffer(8)
+            return direct_read(buf), typed_read(buf)
+
+        t, wa = self.translate(f, [str])
+        # check that the effect of direct_read
+        LIST = LIST_OF(lltype.Char)
+        PLIST = lltype.Ptr(LIST)
+        direct_effects = self._analyze_graph(t, wa, direct_read)
+        assert direct_effects == frozenset([
+            ('readstruct', PLIST, 'length'),
+            ('readstruct', PLIST, 'items'),
+            ('readarray', LIST.items),
+        ])
+
+        # typed_read contains many effects because it reads the vtable etc.,
+        # but we want to check that it contains also the expected effects
+        typed_effects = self._analyze_graph(t, wa, typed_read)
+        expected = frozenset([
+            ('readstruct', PLIST, 'items'),
+            ('readarray', LIST.items),
+        ])
+        assert expected.issubset(typed_effects)
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -157,5 +157,7 @@
         elif isinstance(ofs, llmemory.FieldOffset):
             T = ofs.TYPE
             return ('readinteriorfield', lltype.Ptr(T), ofs.fldname)
+        elif isinstance(ofs, llmemory.ArrayItemsOffset):
+            return ('readarray', lltype.Ptr(ofs.TYPE))
         else:
             assert False, 'implement me'

From pypy.commits at gmail.com  Mon May 22 13:03:53 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 10:03:53 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: implement the analyzer for
 gc_store_indexed
Message-ID: <592319f9.6896df0a.ffe85.f1e5@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91376:1e87cd8c5f6c
Date: 2017-05-22 18:59 +0200
http://bitbucket.org/pypy/pypy/changeset/1e87cd8c5f6c/

Log:	implement the analyzer for gc_store_indexed

diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -418,6 +418,10 @@
         result = wa.analyze(graph.startblock.operations[-1])
         return result
 
+    def _filter_reads(self, effects):
+        result = [item for item in effects if not item[0].startswith('read')]
+        return frozenset(result)
+
     def test_gc_load_indexed_str(self):
         from rpython.rlib.buffer import StringBuffer
 
@@ -476,3 +480,28 @@
             ('readarray', LIST.items),
         ])
         assert expected.issubset(typed_effects)
+
+    def test_gc_store_indexed_str(self):
+        from rpython.rlib.mutbuffer import MutableStringBuffer
+
+        def typed_write(buf):
+            return buf.typed_write(lltype.Signed, 0, 42)
+
+        def direct_write(buf):
+            return buf.setitem(0, 'A')
+
+        def f(x):
+            buf = MutableStringBuffer(8)
+            return direct_write(buf), typed_write(buf)
+
+        t, wa = self.translate(f, [str])
+        # check that the effect of direct_write
+        direct_effects = self._analyze_graph(t, wa, direct_write)
+        direct_effects = self._filter_reads(direct_effects)
+        assert direct_effects == frozenset([
+            ('interiorfield', lltype.Ptr(STR), 'chars')
+        ])
+        #
+        typed_effects = self._analyze_graph(t, wa, typed_write)
+        typed_effects = self._filter_reads(typed_effects)
+        assert typed_effects == direct_effects
diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -63,7 +63,8 @@
                 name = self._getinteriorname(op)
                 return self._interiorfield_result(op.args[0].concretetype, name)
         elif op.opname == "gc_store_indexed":
-            assert False, 'implement me'
+            if graphinfo is None or not graphinfo.is_fresh_malloc(op.args[0]):
+                return self._gc_store_indexed_result(op)
         return empty_set
 
     def _array_result(self, TYPE):
@@ -72,6 +73,39 @@
     def _interiorfield_result(self, TYPE, fieldname):
         return frozenset([("interiorfield", TYPE, fieldname)])
 
+    def _gc_store_indexed_result(self, op):
+        base_ofs = op.args[4].value
+        effect = self._get_effect_for_offset(base_ofs)
+        return frozenset([effect])
+
+    def _get_effect_for_offset(self, ofs, prefix=''):
+        # gc_{load,store}_indexed are generic operation which operate on
+        # various data types: depending on the symbolic offset, they can be
+        # equivalent as reading/writing an array, and interiorfield, etc. The
+        # following logic tries to catch all the known usages. If you see an
+        # 'implement me', please update the logic accordingly
+        if isinstance(ofs, llmemory.CompositeOffset):
+            # get the effect for the first component and modify it if
+            # necessary
+            sub_offsets = ofs.offsets
+            effect = self._get_effect_for_offset(sub_offsets[0])
+            for sub_ofs in sub_offsets[1:]:
+                if isinstance(sub_ofs, llmemory.ArrayItemsOffset):
+                    # reading from the middle of an array is the same as
+                    # reading from the beginning, so we don't need to change
+                    # the effect
+                    pass
+                else:
+                    assert False, 'implement me'
+            return effect
+        elif isinstance(ofs, llmemory.FieldOffset):
+            T = ofs.TYPE
+            return (prefix + 'interiorfield', lltype.Ptr(T), ofs.fldname)
+        elif isinstance(ofs, llmemory.ArrayItemsOffset):
+            return (prefix + 'array', lltype.Ptr(ofs.TYPE))
+        else:
+            assert False, 'implement me'
+
     def compute_graph_info(self, graph):
         return FreshMallocs(graph)
 
@@ -129,35 +163,8 @@
             return self._gc_load_store_result(op)
         return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo)
 
-    def _gc_load_store_result(self, op):
+    def _gc_load_indexed_result(self, op):
         base_offset = op.args[3].value
-        effect = self._get_effect_for_offset(base_offset)
+        effect = self._get_effect_for_offset(base_offset, prefix='read')
         return frozenset([effect])
 
-    def _get_effect_for_offset(self, ofs):
-        # gc_{load,store}_indexed are generic operation which operate on
-        # various data types: depending on the symbolic offset, they can be
-        # equivalent as reading/writing an array, and interiorfield, etc. The
-        # following logic tries to catch all the known usages. If you see an
-        # 'implement me', please update the logic accordingly
-        if isinstance(ofs, llmemory.CompositeOffset):
-            # get the effect for the first component and modify it if
-            # necessary
-            sub_offsets = ofs.offsets
-            effect = self._get_effect_for_offset(sub_offsets[0])
-            for sub_ofs in sub_offsets[1:]:
-                if isinstance(sub_ofs, llmemory.ArrayItemsOffset):
-                    # reading from the middle of an array is the same as
-                    # reading from the beginning, so we don't need to change
-                    # the effect
-                    pass
-                else:
-                    assert False, 'implement me'
-            return effect
-        elif isinstance(ofs, llmemory.FieldOffset):
-            T = ofs.TYPE
-            return ('readinteriorfield', lltype.Ptr(T), ofs.fldname)
-        elif isinstance(ofs, llmemory.ArrayItemsOffset):
-            return ('readarray', lltype.Ptr(ofs.TYPE))
-        else:
-            assert False, 'implement me'

From pypy.commits at gmail.com  Mon May 22 13:03:55 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 10:03:55 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: add a passing test for
 gc_store_indexed on a list of chars
Message-ID: <592319fb.5788df0a.610a9.b81a@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91377:64c71c919e55
Date: 2017-05-22 19:01 +0200
http://bitbucket.org/pypy/pypy/changeset/64c71c919e55/

Log:	add a passing test for gc_store_indexed on a list of chars

diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -505,3 +505,29 @@
         typed_effects = self._analyze_graph(t, wa, typed_write)
         typed_effects = self._filter_reads(typed_effects)
         assert typed_effects == direct_effects
+
+    def test_gc_store_indexed_list_of_chars(self):
+        from rpython.rlib.buffer import ByteBuffer
+
+        def typed_write(buf):
+            return buf.typed_write(lltype.Signed, 0, 42)
+
+        def direct_write(buf):
+            return buf.setitem(0, 'A')
+
+        def f(x):
+            buf = ByteBuffer(8)
+            return direct_write(buf), typed_write(buf)
+
+        t, wa = self.translate(f, [str])
+        # check that the effect of direct_write
+        LIST = LIST_OF(lltype.Char)
+        direct_effects = self._analyze_graph(t, wa, direct_write)
+        direct_effects = self._filter_reads(direct_effects)
+        assert direct_effects == frozenset([
+            ('array', LIST.items),
+        ])
+        #
+        typed_effects = self._analyze_graph(t, wa, typed_write)
+        typed_effects = self._filter_reads(typed_effects)
+        assert typed_effects == direct_effects

From pypy.commits at gmail.com  Mon May 22 14:56:17 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 22 May 2017 11:56:17 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: backed out 552622605578,
 breaks too many things
Message-ID: <59233451.59b0df0a.e5a49.0228@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91378:ff2ce5a214bd
Date: 2017-05-22 21:45 +0300
http://bitbucket.org/pypy/pypy/changeset/ff2ce5a214bd/

Log:	backed out 552622605578, breaks too many things

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -272,7 +272,6 @@
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCWrapperObject),
     __repr__ = interp2app(W_PyCWrapperObject.descr_method_repr),
     # XXX missing: __getattribute__
-    __hash__ = None
     )
 W_PyCWrapperObject.typedef.acceptable_as_base_class = False
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -159,11 +159,3 @@
         fd = BytesIO()
         # only test that it works
         fd.write(a)
-
-    def test_hash(self):
-        module = self.import_module(name='array')
-        a = module.array('c')
-        exc = raises(TypeError, hash, a)
-        assert 'unhashable' in str(exc.value)
-        exc = raises(TypeError, hash, a.__mul__)
-        assert 'unhashable' in str(exc.value)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -338,6 +338,7 @@
                 setattr(struct, slot_names[1], slot_func_helper)
 
 def add_operators(space, dict_w, pto):
+    # XXX support PyObject_HashNotImplemented
     for method_name, slot_names, wrapper_func, wrapper_func_kwds, doc in slotdefs_for_wrappers:
         if method_name in dict_w:
             continue
@@ -364,8 +365,6 @@
             rffi.charp2str(cts.cast('char*', pto.c_tp_doc)))
     if pto.c_tp_new:
         add_tp_new_wrapper(space, dict_w, pto)
-    if not pto.c_tp_hash:
-        dict_w['__hash__'] = space.w_None
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def tp_new_wrapper(space, self, w_args, w_kwds):
@@ -944,7 +943,6 @@
     if w_obj.is_cpytype():
         Py_DecRef(space, pto.c_tp_dict)
     w_dict = w_obj.getdict(space)
-
     # pass in the w_obj to convert any values that are
     # unbound GetSetProperty into bound PyGetSetDescrObject
     pto.c_tp_dict = make_ref(space, w_dict, w_obj)

From pypy.commits at gmail.com  Mon May 22 18:23:04 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 15:23:04 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: @enforceargs does not play
 well with @specialize, use _annenforceargs_ directly
Message-ID: <592364c8.519c1c0a.97996.0204@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91379:616c5957da8d
Date: 2017-05-23 00:19 +0200
http://bitbucket.org/pypy/pypy/changeset/616c5957da8d/

Log:	@enforceargs does not play well with @specialize, use
	_annenforceargs_ directly

diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -5,8 +5,7 @@
 from rpython.rtyper.annlowlevel import llhelper, MixLevelHelperAnnotator
 from rpython.rtyper.annlowlevel import hlstr, hlunicode
 from rpython.rtyper.llannotation import lltype_to_annotation
-from rpython.rlib.objectmodel import (we_are_translated, specialize, compute_hash,
-                                      enforceargs)
+from rpython.rlib.objectmodel import we_are_translated, specialize, compute_hash
 from rpython.jit.metainterp import history, compile
 from rpython.jit.metainterp.optimize import SpeculativeError
 from rpython.jit.codewriter import heaptracker, longlong
@@ -495,9 +494,9 @@
         return llop.raw_load(longlong.FLOATSTORAGE, gcref, ofs)
 
     @specialize.argtype(1)
-    @enforceargs(newvalue=longlong.r_float_storage)
     def write_float_at_mem(self, gcref, ofs, newvalue):
         llop.raw_store(lltype.Void, gcref, ofs, newvalue)
+    write_float_at_mem._annenforceargs_ = [None, None, None, longlong.r_float_storage]
 
     # ____________________________________________________________
 

From pypy.commits at gmail.com  Mon May 22 18:23:06 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 15:23:06 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: typo
Message-ID: <592364ca.5987df0a.4d9bb.bfa3@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91380:a7c92808278f
Date: 2017-05-23 00:19 +0200
http://bitbucket.org/pypy/pypy/changeset/a7c92808278f/

Log:	typo

diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -160,7 +160,7 @@
             return frozenset([("readinteriorfield", op.args[0].concretetype,
                             name)])
         elif op.opname == "gc_load_indexed":
-            return self._gc_load_store_result(op)
+            return self._gc_load_indexed_result(op)
         return WriteAnalyzer.analyze_simple_operation(self, op, graphinfo)
 
     def _gc_load_indexed_result(self, op):

From pypy.commits at gmail.com  Mon May 22 18:27:19 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 15:27:19 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: fix test_gc_load_indexed_str
Message-ID: <592365c7.53afdf0a.6b4d0.8be9@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91381:ad1105da8697
Date: 2017-05-23 00:26 +0200
http://bitbucket.org/pypy/pypy/changeset/ad1105da8697/

Log:	fix test_gc_load_indexed_str

diff --git a/rpython/translator/backendopt/writeanalyze.py b/rpython/translator/backendopt/writeanalyze.py
--- a/rpython/translator/backendopt/writeanalyze.py
+++ b/rpython/translator/backendopt/writeanalyze.py
@@ -88,7 +88,7 @@
             # get the effect for the first component and modify it if
             # necessary
             sub_offsets = ofs.offsets
-            effect = self._get_effect_for_offset(sub_offsets[0])
+            effect = self._get_effect_for_offset(sub_offsets[0], prefix)
             for sub_ofs in sub_offsets[1:]:
                 if isinstance(sub_ofs, llmemory.ArrayItemsOffset):
                     # reading from the middle of an array is the same as

From pypy.commits at gmail.com  Mon May 22 18:30:50 2017
From: pypy.commits at gmail.com (antocuni)
Date: Mon, 22 May 2017 15:30:50 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: hg merge default
Message-ID: <5923669a.430d1c0a.5b3fa.06dd@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91382:4d0500d6eced
Date: 2017-05-23 00:30 +0200
http://bitbucket.org/pypy/pypy/changeset/4d0500d6eced/

Log:	hg merge default

diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,9 +26,9 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
-WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+WIN64 = sys.platform == 'win32' and sys.maxint == 2 ** 63 - 1
 
 
 def get_com_error(errcode, riid, pIunk):
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -94,14 +97,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -126,27 +124,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -156,7 +153,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -168,15 +165,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -188,7 +188,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -201,7 +201,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -222,10 +222,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -259,7 +257,7 @@
         if (sys.platform == 'win32' and isinstance(argument, (int, long))
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -281,6 +279,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -317,7 +316,7 @@
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
-                print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
+                print >> sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
             if self._restype_ is not None:
                 return res
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -220,9 +220,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -39,3 +39,20 @@
 
 Internal refactoring of buffers and memoryviews. Memoryviews will now be
 accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -297,20 +297,19 @@
         except SocketError as e:
             raise converted_error(space, e)
 
-    @unwrap_spec(level=int, optname=int)
-    def getsockopt_w(self, space, level, optname, w_buflen=None):
+    @unwrap_spec(level=int, optname=int, buflen=int)
+    def getsockopt_w(self, space, level, optname, buflen=0):
         """getsockopt(level, option[, buffersize]) -> value
 
         Get a socket option.  See the Unix manual for level and option.
         If a nonzero buffersize argument is given, the return value is a
         string of that length; otherwise it is an integer.
         """
-        if w_buflen is None:
+        if buflen == 0:
             try:
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
-        buflen = space.int_w(w_buflen)
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -582,6 +582,19 @@
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
 
+    def test_getsetsockopt_zero(self):
+        # related to issue #2561: when specifying the buffer size param:
+        # if 0 or None, should return the setted value,
+        # otherwise an empty buffer of the specified size
+        import _socket
+        s = _socket.socket()
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1557,7 +1557,6 @@
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null):
-    from pypy.module.cpyext.pyobject import make_ref, from_ref
     from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
     from pypy.module.cpyext.pyobject import get_w_obj_and_decref
     from pypy.module.cpyext.pyerrors import PyErr_Occurred
diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py
--- a/pypy/module/cpyext/eval.py
+++ b/pypy/module/cpyext/eval.py
@@ -1,6 +1,8 @@
 from pypy.interpreter.error import oefmt
 from pypy.interpreter.astcompiler import consts
 from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.rarithmetic import widen
 from pypy.module.cpyext.api import (
     cpython_api, CANNOT_FAIL, CONST_STRING, FILEP, fread, feof, Py_ssize_tP,
     cpython_struct, is_valid_fp)
@@ -227,4 +229,51 @@
     cf.c_cf_flags = rffi.cast(rffi.INT, flags)
     return result
 
+ at cpython_api([], rffi.INT_real, error=CANNOT_FAIL)
+def Py_GetRecursionLimit(space):
+    from pypy.module.sys.vm import getrecursionlimit
+    return space.int_w(getrecursionlimit(space))
 
+ at cpython_api([rffi.INT_real], lltype.Void, error=CANNOT_FAIL)
+def Py_SetRecursionLimit(space, limit):
+    from pypy.module.sys.vm import setrecursionlimit
+    setrecursionlimit(space, widen(limit))
+
+limit = 0 # for testing
+
+ at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
+def Py_EnterRecursiveCall(space, where):
+    """Marks a point where a recursive C-level call is about to be performed.
+
+    If USE_STACKCHECK is defined, this function checks if the the OS
+    stack overflowed using PyOS_CheckStack().  In this is the case, it
+    sets a MemoryError and returns a nonzero value.
+
+    The function then checks if the recursion limit is reached.  If this is the
+    case, a RuntimeError is set and a nonzero value is returned.
+    Otherwise, zero is returned.
+
+    where should be a string such as " in instance check" to be
+    concatenated to the RuntimeError message caused by the recursion depth
+    limit."""
+    if not we_are_translated():
+        # XXX hack since the stack checks only work translated
+        global limit
+        limit += 1
+        if limit > 10:
+            raise oefmt(space.w_RuntimeError, 
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+        return 0
+    from rpython.rlib.rstack import stack_almost_full
+    if stack_almost_full():
+        raise oefmt(space.w_RuntimeError,
+                 "maximum recursion depth exceeded%s", rffi.charp2str(where))
+    return 0
+
+ at cpython_api([], lltype.Void)
+def Py_LeaveRecursiveCall(space):
+    """Ends a Py_EnterRecursiveCall().  Must be called once for each
+    successful invocation of Py_EnterRecursiveCall()."""
+    # A NOP in PyPy
+    if not we_are_translated():
+        limit = 0
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,1 @@
-#define PyList_GET_ITEM(o, i) PyList_GetItem((PyObject*)(o), (i))
+/* empty */
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -1,9 +1,10 @@
 
+from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
                                     build_type_checkers)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
-from pypy.module.cpyext.pyobject import Py_DecRef, PyObject, make_ref
+from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
@@ -19,59 +20,69 @@
     PySequence_SetItem()  or expose the object to Python code before
     setting all items to a real object with PyList_SetItem().
     """
-    return space.newlist([None] * len)
+    w_list = space.newlist([None] * len)
+    #w_list.convert_to_cpy_strategy(space)
+    return w_list
 
- at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], PyObject, error=CANNOT_FAIL,
-             result_borrowed=True)
-def PyList_SET_ITEM(space, w_list, index, w_item):
-    """Macro form of PyList_SetItem() without error checking. This is normally
+ at always_inline
+def get_list_storage(space, w_list):
+    from pypy.module.cpyext.sequence import CPyListStrategy
+    w_list.convert_to_cpy_strategy(space)
+    return CPyListStrategy.unerase(w_list.lstorage)
+
+ at cpython_api([rffi.VOIDP, Py_ssize_t, PyObject], lltype.Void, error=CANNOT_FAIL)
+def PyList_SET_ITEM(space, w_list, index, py_item):
+    """Form of PyList_SetItem() without error checking. This is normally
     only used to fill in new lists where there is no previous content.
 
-    This function "steals" a reference to item, and, unlike PyList_SetItem(),
-    does not discard a reference to any item that it being replaced; any
-    reference in list at position i will be leaked.
+    "steals" a reference to item, and, unlike PyList_SetItem(), does not
+    discard a reference to any item that it being replaced; any reference in
+    list at position i will be leaked.
     """
     assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
     assert 0 <= index < w_list.length()
-    # Deliberately leak, so that it can be safely decref'd.
-    make_ref(space, w_list.getitem(index))
-    Py_DecRef(space, w_item)
-    w_list.setitem(index, w_item)
-    return w_item
-
+    storage._elems[index] = py_item
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
-def PyList_SetItem(space, w_list, index, w_item):
+def PyList_SetItem(space, w_list, index, py_item):
     """Set the item at index index in list to item.  Return 0 on success
     or -1 on failure.
-    
+
     This function "steals" a reference to item and discards a reference to
     an item already in the list at the affected position.
     """
-    Py_DecRef(space, w_item)
     if not isinstance(w_list, W_ListObject):
+        decref(space, py_item)
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
+        decref(space, py_item)
         raise oefmt(space.w_IndexError, "list assignment index out of range")
-    w_list.setitem(index, w_item)
+    storage = get_list_storage(space, w_list)
+    py_old = storage._elems[index]
+    storage._elems[index] = py_item
+    decref(w_list.space, py_old)
     return 0
 
- at cpython_api([PyObject, Py_ssize_t], PyObject, result_borrowed=True)
+ at cpython_api([rffi.VOIDP, Py_ssize_t], PyObject, result_is_ll=True)
+def PyList_GET_ITEM(space, w_list, index):
+    assert isinstance(w_list, W_ListObject)
+    storage = get_list_storage(space, w_list)
+    assert 0 <= index < w_list.length()
+    return storage._elems[index]     # borrowed ref
+
+ at cpython_api([PyObject, Py_ssize_t], PyObject, result_is_ll=True)
 def PyList_GetItem(space, w_list, index):
     """Return the object at position pos in the list pointed to by p.  The
     position must be positive, indexing from the end of the list is not
     supported.  If pos is out of bounds, return NULL and set an
     IndexError exception."""
-    from pypy.module.cpyext.sequence import CPyListStrategy
     if not isinstance(w_list, W_ListObject):
         PyErr_BadInternalCall(space)
     if index < 0 or index >= w_list.length():
         raise oefmt(space.w_IndexError, "list index out of range")
-    cpy_strategy = space.fromcache(CPyListStrategy)
-    if w_list.strategy is not cpy_strategy:
-        w_list.ensure_object_strategy() # make sure we can return a borrowed obj
-    w_res = w_list.getitem(index)
-    return w_res     # borrowed ref
+    storage = get_list_storage(space, w_list)
+    return storage._elems[index]     # borrowed ref
 
 
 @cpython_api([PyObject, PyObject], rffi.INT_real, error=-1)
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -30,8 +30,7 @@
         return subtype_dealloc.api_func
 
     def allocate(self, space, w_type, itemcount=0):
-        # similar to PyType_GenericAlloc?
-        # except that it's not related to any pypy object.
+        # typically called from PyType_GenericAlloc via typedescr.allocate
         # this returns a PyObject with ob_refcnt == 1.
 
         pytype = as_pyobj(space, w_type)
@@ -250,6 +249,8 @@
     w_obj = rawrefcount.to_obj(W_Root, pyobj)
     return w_obj is not None and w_obj is not w_marker_deallocating
 
+def w_obj_has_pyobj(w_obj):
+    return bool(rawrefcount.from_obj(PyObject, w_obj))
 
 def is_pyobj(x):
     if x is None or isinstance(x, W_Root):
@@ -275,13 +276,14 @@
     """
     if is_pyobj(obj):
         pyobj = rffi.cast(PyObject, obj)
+        at_least = 1
     else:
         pyobj = as_pyobj(space, obj, w_userdata)
+        at_least = rawrefcount.REFCNT_FROM_PYPY
     if pyobj:
-        assert pyobj.c_ob_refcnt > 0
+        assert pyobj.c_ob_refcnt >= at_least
         pyobj.c_ob_refcnt += 1
-        if not is_pyobj(obj):
-            keepalive_until_here(obj)
+        keepalive_until_here(obj)
     return pyobj
 
 
@@ -315,9 +317,14 @@
         obj = rffi.cast(PyObject, obj)
         if obj:
             assert obj.c_ob_refcnt > 0
+            assert obj.c_ob_pypy_link == 0 or obj.c_ob_refcnt > rawrefcount.REFCNT_FROM_PYPY
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            #else:
+            #    w_obj = rawrefcount.to_obj(W_Root, ref)
+            #    if w_obj is not None:
+            #        assert obj.c_ob_refcnt >= rawrefcount.REFCNT_FROM_PYPY
     else:
         get_w_obj_and_decref(space, obj)
 
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -256,8 +256,9 @@
     def setitem(self, w_list, index, w_obj):
         storage = self.unerase(w_list.lstorage)
         index = self._check_index(index, storage._length)
-        decref(w_list.space, storage._elems[index])
+        py_old = storage._elems[index]
         storage._elems[index] = make_ref(w_list.space, w_obj)
+        decref(w_list.space, py_old)
 
     def length(self, w_list):
         storage = self.unerase(w_list.lstorage)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -490,29 +490,6 @@
     0 on success, -1 on failure."""
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], rffi.INT_real, error=1)
-def Py_EnterRecursiveCall(space, where):
-    """Marks a point where a recursive C-level call is about to be performed.
-
-    If USE_STACKCHECK is defined, this function checks if the the OS
-    stack overflowed using PyOS_CheckStack().  In this is the case, it
-    sets a MemoryError and returns a nonzero value.
-
-    The function then checks if the recursion limit is reached.  If this is the
-    case, a RuntimeError is set and a nonzero value is returned.
-    Otherwise, zero is returned.
-
-    where should be a string such as " in instance check" to be
-    concatenated to the RuntimeError message caused by the recursion depth
-    limit."""
-    raise NotImplementedError
-
- at cpython_api([], lltype.Void)
-def Py_LeaveRecursiveCall(space):
-    """Ends a Py_EnterRecursiveCall().  Must be called once for each
-    successful invocation of Py_EnterRecursiveCall()."""
-    raise NotImplementedError
-
 @cpython_api([PyFileObject], lltype.Void)
 def PyFile_IncUseCount(space, p):
     """Increments the PyFileObject's internal use count to indicate
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -1872,6 +1872,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj1, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
@@ -1887,6 +1888,7 @@
             for (nn = 0; nn < n; nn++)
             {
                 v = PyList_GetItem(obj2, nn);
+                Py_INCREF(v);
                 PyList_SetItem(ret, nn+ii*n, v);
             }
         return ret;
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -331,3 +331,30 @@
                 def nested_flags():
                     return module.get_flags()""" in ns
         assert ns['nested_flags']() == (1, 0x2000)  # CO_FUTURE_DIVISION
+
+    def test_recursive_function(self):
+        module = self.import_extension('foo', [
+            ("call_recursive", "METH_NOARGS",
+             """
+                int res = 0;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse"))
+                        return -1;
+                    res ++;
+                    return recurse();
+                }
+                int oldlimit = Py_GetRecursionLimit();
+                Py_SetRecursionLimit(200);
+                res = recurse();
+                Py_SetRecursionLimit(oldlimit);
+                if (PyErr_Occurred())
+                    return NULL;
+                return PyLong_FromLong(res);
+             """),], prologue= ''' int recurse(void); '''
+            )
+        try:
+            res = module.call_recursive()
+        except RuntimeError as e:
+            assert 'while calling recurse' in str(e)
+        else:
+            assert False, "expected RuntimeError"  
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -38,11 +38,13 @@
         assert api.PyList_Insert(w_l, 0, space.wrap(1)) == 0
         assert api.PyList_Size(w_l) == 3
         assert api.PyList_Insert(w_l, 99, space.wrap(2)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 2
+        assert api.PyObject_Compare(api.PyList_GetItem(w_l, 3),
+                                    space.wrap(2)) == 0
         # insert at index -1: next-to-last
         assert api.PyList_Insert(w_l, -1, space.wrap(3)) == 0
-        assert space.unwrap(api.PyList_GetItem(w_l, 3)) == 3
-    
+        assert api.PyObject_Compare(api.PyList_GET_ITEM(w_l, 3),
+                                    space.wrap(3)) == 0
+
     def test_sort(self, space, api):
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
@@ -152,29 +154,35 @@
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
-            ("test_macro_invocations", "METH_NOARGS",
+            ("test_macro_invocations", "METH_O",
              """
              PyObject* o = PyList_New(2);
              PyListObject* l = (PyListObject*)o;
 
+             Py_INCREF(args);
+             Py_INCREF(args);
+             PyList_SET_ITEM(o, 0, args);
+             PyList_SET_ITEM(l, 1, args);
 
-             Py_INCREF(o);
-             PyList_SET_ITEM(o, 0, o);
-             Py_INCREF(o);
-             PyList_SET_ITEM(l, 1, o);
+             if(PyList_GET_ITEM(o, 0) != PyList_GET_ITEM(l, 1))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_ITEM error");
+                return NULL;
+             }
 
-             PyList_GET_ITEM(o, 0);
-             PyList_GET_ITEM(l, 1);
-
-             PyList_GET_SIZE(o);
-             PyList_GET_SIZE(l);
+             if(PyList_GET_SIZE(o) != PyList_GET_SIZE(l))
+             {
+                PyErr_SetString(PyExc_AssertionError, "PyList_GET_SIZE error");
+                return NULL;
+             }
 
              return o;
              """
             )
         ])
-        x = module.test_macro_invocations()
-        assert x[0] is x[1] is x
+        s = 'abcdef'
+        x = module.test_macro_invocations(s)
+        assert x[0] is x[1] is s
 
     def test_get_item_macro(self):
         module = self.import_extension('foo', [
@@ -183,39 +191,96 @@
                 PyObject* o, *o2, *o3;
                 o = PyList_New(1);
 
-                o2 = PyInt_FromLong(0);
+                o2 = PyBytes_FromString("test_get_item0");
+                Py_INCREF(o2);
                 PyList_SET_ITEM(o, 0, o2);
-                o2 = NULL;
 
                 o3 = PyList_GET_ITEM(o, 0);
                 Py_INCREF(o3);
-                Py_CLEAR(o);
+                Py_DECREF(o);
+                Py_DECREF(o2);
                 return o3;
              """)])
-        assert module.test_get_item() == 0
+        assert module.test_get_item() == b'test_get_item0'
 
-    def test_set_item_macro(self):
+    def test_item_refcounts(self):
         """PyList_SET_ITEM leaks a reference to the target."""
         module = self.import_extension('foo', [
-             ("test_refcount_diff_after_setitem", "METH_NOARGS",
+             ("test_refcount_diff", "METH_VARARGS",
              """
-                PyObject* o = PyList_New(0);
-                PyObject* o2 = PyList_New(0);
-                Py_ssize_t refcount, new_refcount;
+                /* test that the refcount differences for functions
+                 * are correct. diff1 - expected refcount diff for i1,
+                 *              diff2 - expected refcount diff for i2
+                 */
+                #define CHECKCOUNT(diff1, diff2, action) \
+                    new_count1 = Py_REFCNT(i1); \
+                    new_count2 = Py_REFCNT(i2); \
+                    diff = new_count1 - old_count1; \
+                    if (diff != diff1) {\
+                        sprintf(errbuffer, action \
+                            " i1 expected diff of %ld got %ld", (long)diff1, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    diff = new_count2 - old_count2; \
+                    if (diff != diff2) {\
+                        sprintf(errbuffer, action \
+                            " i2 expected diff of %ld got %ld", (long)diff2, (long)diff); \
+                    PyErr_SetString(PyExc_AssertionError, errbuffer); \
+                    return NULL; } \
+                    old_count1 = new_count1; \
+                    old_count2 = new_count2;
 
-                PyList_Append(o, o2);  // does not steal o2
+                PyObject* tmp, *o = PyList_New(0);
+                char errbuffer[1024];
+                PyObject* i1 = PyTuple_GetItem(args, 0);
+                PyObject* i2 = PyTuple_GetItem(args, 1);
+                Py_ssize_t old_count1, new_count1;
+                Py_ssize_t old_count2, new_count2;
+                Py_ssize_t diff;
+                int ret;
 
-                refcount = Py_REFCNT(o2);
+                old_count1 = Py_REFCNT(i1);
+                old_count2 = Py_REFCNT(i2);
 
-                // Steal a reference to o2, but leak the old reference to o2.
-                // The net result should be no change in refcount.
-                PyList_SET_ITEM(o, 0, o2);
+                ret = PyList_Append(o, i1);
+                if (ret != 0) 
+                    return NULL;
+                /* check the result of Append(), and also force the list
+                   to use the CPyListStrategy now */
+                if (PyList_GET_ITEM(o, 0) != i1)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "Append() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(1, 0, "PyList_Append");
 
-                new_refcount = Py_REFCNT(o2);
+                Py_INCREF(i2);   /* for PyList_SET_ITEM */
+                CHECKCOUNT(0, 1, "Py_INCREF");
 
-                Py_CLEAR(o);
-                Py_DECREF(o2); // append incref'd.
-                // Py_CLEAR(o2);  // naive implementation would fail here.
-                return PyLong_FromSsize_t(new_refcount - refcount);
+                PyList_SET_ITEM(o, 0, i2);
+                CHECKCOUNT(0, 0, "PyList_SET_ITEM");
+
+                tmp = PyList_GET_ITEM(o, 0);
+                if (tmp != i2)
+                {
+                    PyErr_SetString(PyExc_AssertionError, "SetItem() error?");
+                    return NULL;
+                }
+                CHECKCOUNT(0, 0, "PyList_GET_ITEM");
+
+                PyList_SetItem(o, 0, i1);
+                CHECKCOUNT(0, -1, "PyList_Set_Item");
+
+                PyList_GetItem(o, 0);
+                CHECKCOUNT(0, 0, "PyList_Get_Item");
+
+                Py_DECREF(o); 
+                #ifndef PYPY_VERSION
+                {
+                    // PyPy deletes only at teardown
+                    CHECKCOUNT(-1, 0, "Py_DECREF(o)");
+                }
+                #endif
+                return PyLong_FromSsize_t(0);
              """)])
-        assert module.test_refcount_diff_after_setitem() == 0
+        assert module.test_refcount_diff(["first"], ["second"]) == 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -24,7 +24,7 @@
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
-    PyObject, make_ref, create_ref, from_ref, get_typedescr, make_typedescr,
+    PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
     track_reference, Py_DecRef, as_pyobj)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/README b/pypy/module/test_lib_pypy/ctypes_tests/README
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/ctypes_tests/README
@@ -0,0 +1,8 @@
+-------Ctypes tests------
+
+Unlike the other tests in the PyPy sources, these tests are assumed to run after the translation is complete.
+Therefore, using the resulting binary, you can then call, for example:
+
+/path/to/your_modified_pypy_binary pytest.py pypy/module/test_lib_pypy/ctypes_tests/test_libc.py
+
+This also applies to any other test in ctypes_tests.
\ No newline at end of file
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py b/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
deleted file mode 100644
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_fastpath.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from ctypes import CDLL, POINTER, pointer, c_byte, c_int, c_char_p, CFUNCTYPE, c_void_p, c_size_t
-import sys
-import py
-from support import BaseCTypesTestChecker
-
-class MyCDLL(CDLL):
-    def __getattr__(self, attr):
-        fn = self[attr] # this way it's not cached as an attribute
-        fn._slowpath_allowed = False
-        return fn
-
-def setup_module(mod):
-    import conftest
-    _ctypes_test = str(conftest.sofile)
-    mod.dll = MyCDLL(_ctypes_test)  # slowpath not allowed
-    mod.dll2 = CDLL(_ctypes_test)   # slowpath allowed
-
-
-class TestFastpath(BaseCTypesTestChecker):
-
-    def test_fastpath_forbidden(self):
-        def myfunc():
-            pass
-        #
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        #
-        # so far, it's still using the slowpath
-        assert not tf_b._is_fastpath
-        tf_b.callable = myfunc
-        tf_b.argtypes = (c_byte,)
-        # errcheck prevented the fastpath to kick in
-        assert not tf_b._is_fastpath
-        #
-        del tf_b.callable
-        tf_b.argtypes = (c_byte,) # try to re-enable the fastpath
-        assert tf_b._is_fastpath
-        #
-        assert not tf_b._slowpath_allowed
-        py.test.raises(AssertionError, "tf_b.callable = myfunc")
-        py.test.raises(AssertionError, "tf_b('aaa')") # force a TypeError
-
-    def test_simple_args(self):
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        assert tf_b(-126) == -42
-
-    def test_from_cfunctype(self):
-        from _ctypes import _memmove_addr
-        functype = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)
-        my_memmove = functype(_memmove_addr)
-        assert my_memmove._is_fastpath
-
-    def test_undeclared_restype(self):
-        # make sure we get a fresh function
-        try:
-            del dll.tf_i
-        except AttributeError:
-            pass
-        tf_i = dll.tf_i
-        assert not tf_i._is_fastpath
-        tf_i.argtypes = (c_int,)
-        assert tf_i._is_fastpath
-        assert tf_i(12) == 4
-
-    def test_pointer_args(self):
-        f = dll._testfunc_p_p
-        f.restype = POINTER(c_int)
-        f.argtypes = [POINTER(c_int)]
-        v = c_int(42)
-        result = f(pointer(v))
-        assert type(result) == POINTER(c_int)
-        assert result.contents.value == 42
-
-    def test_simple_pointer_args(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        mystr = c_char_p("abcd")
-        result = f(mystr, ord("b"))
-        assert result == "bcd"
-
-    def test_strings(self):
-        f = dll.my_strchr
-        f.argtypes = [c_char_p, c_int]
-        f.restype = c_char_p
-        result = f("abcd", ord("b"))
-        assert result == "bcd"
-
-    def test_errcheck(self):
-        def errcheck(result, func, args):
-            return 'hello'
-        tf_b = dll.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.errcheck = errcheck
-        assert tf_b(-126) == 'hello'
-
-    def test_array_to_ptr(self):
-        ARRAY = c_int * 8
-        func = dll._testfunc_ai8
-        func.restype = POINTER(c_int)
-        func.argtypes = [ARRAY]
-        array = ARRAY(1, 2, 3, 4, 5, 6, 7, 8)
-        ptr = func(array)
-        assert ptr[0] == 1
-        assert ptr[7] == 8
-
-
-class TestFallbackToSlowpath(BaseCTypesTestChecker):
-
-    def test_argtypes_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_char_p,)  # this is intentionally wrong
-        tf_b.argtypes = None # kill the fast path
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -42
-
-    def test_callable_is_None(self):
-        tf_b = dll2.tf_b
-        tf_b.restype = c_byte
-        tf_b.argtypes = (c_byte,)
-        tf_b.callable = lambda x: x+1
-        assert not tf_b._is_fastpath
-        assert tf_b(-126) == -125
-        tf_b.callable = None
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_functions.py
@@ -478,7 +478,7 @@
         raises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT()))
 
     def test_argument_conversion_and_checks(self):
-        py.test.skip("XXX currently broken on PyPy, sorry")
+        #This test is designed to check for segfaults if the wrong type of argument is passed as parameter
         strlen = dll.my_strchr
         strlen.argtypes = [c_char_p, c_int]
         strlen.restype = c_char_p
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -611,55 +611,31 @@
         return mod_format(space, w_values, self, do_unicode=False)
 
     def descr_eq(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value == w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value == w_other._value)
 
     def descr_ne(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value != w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value != w_other._value)
 
     def descr_lt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value < w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value < w_other._value)
 
     def descr_le(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value <= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value <= w_other._value)
 
     def descr_gt(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value > w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value > w_other._value)
 
     def descr_ge(self, space, w_other):
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            if isinstance(w_other, W_StringBufferObject):
-                return space.newbool(self._value >= w_other.force())
         if not isinstance(w_other, W_BytesObject):
             return space.w_NotImplemented
         return space.newbool(self._value >= w_other._value)
@@ -677,18 +653,6 @@
             from .bytearrayobject import W_BytearrayObject, _make_data
             self_as_bytearray = W_BytearrayObject(_make_data(self._value))
             return space.add(self_as_bytearray, w_other)
-        if space.config.objspace.std.withstrbuf:
-            from pypy.objspace.std.strbufobject import W_StringBufferObject
-            try:
-                other = self._op_val(space, w_other)
-            except OperationError as e:
-                if e.match(space, space.w_TypeError):
-                    return space.w_NotImplemented
-                raise
-            builder = StringBuilder()
-            builder.append(self._value)
-            builder.append(other)
-            return W_StringBufferObject(builder)
         return self._StringMethods_descr_add(space, w_other)
 
     _StringMethods__startswith = _startswith
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -230,15 +230,13 @@
         return list(items)
 
     def switch_to_object_strategy(self):
+        object_strategy = self.space.fromcache(ObjectListStrategy)
+        if self.strategy is object_strategy:
+            return
         list_w = self.getitems()
-        object_strategy = self.space.fromcache(ObjectListStrategy)
         self.strategy = object_strategy
         object_strategy.init_from_list_w(self, list_w)
 
-    def ensure_object_strategy(self):     # for cpyext
-        if self.strategy is not self.space.fromcache(ObjectListStrategy):
-            self.switch_to_object_strategy()
-
     def _temporarily_as_objects(self):
         if self.strategy is self.space.fromcache(ObjectListStrategy):
             return self
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -16,7 +16,7 @@
 from pypy.objspace.std.boolobject import W_BoolObject
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.objspace.std.bytearrayobject import W_BytearrayObject
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
+from pypy.objspace.std.bytesobject import W_BytesObject
 from pypy.objspace.std.complexobject import W_ComplexObject
 from pypy.objspace.std.dictmultiobject import W_DictMultiObject, W_DictObject
 from pypy.objspace.std.floatobject import W_FloatObject
@@ -81,9 +81,6 @@
             W_TypeObject.typedef: W_TypeObject,
             W_UnicodeObject.typedef: W_UnicodeObject,
         }
-        if self.config.objspace.std.withstrbuf:
-            builtin_type_classes[W_BytesObject.typedef] = W_AbstractBytesObject
-
         self.builtin_types = {}
         self._interplevel_classes = {}
         for typedef, cls in builtin_type_classes.items():
@@ -285,7 +282,7 @@
         return W_LongObject.fromint(self, val)
 
     @specialize.argtype(1)
-    def newlong_from_rarith_int(self, val): # val is an rarithmetic type 
+    def newlong_from_rarith_int(self, val): # val is an rarithmetic type
         return W_LongObject.fromrarith_int(val)
 
     def newlong_from_rbigint(self, val):
diff --git a/pypy/objspace/std/strbufobject.py b/pypy/objspace/std/strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/strbufobject.py
+++ /dev/null
@@ -1,96 +0,0 @@
-import inspect
-
-import py
-
-from pypy.objspace.std.bytesobject import W_AbstractBytesObject, W_BytesObject
-from pypy.interpreter.gateway import interp2app, unwrap_spec
-from pypy.interpreter.buffer import SimpleView, StringBuffer
-from pypy.interpreter.error import OperationError
-from rpython.rlib.rstring import StringBuilder
-
-
-class W_StringBufferObject(W_AbstractBytesObject):
-    w_str = None
-
-    def __init__(self, builder):
-        self.builder = builder             # StringBuilder
-        self.length = builder.getlength()
-
-    def force(self):
-        if self.w_str is None:
-            s = self.builder.build()
-            if self.length < len(s):
-                s = s[:self.length]
-            self.w_str = W_BytesObject(s)
-            return s
-        else:
-            return self.w_str._value
-
-    def __repr__(self):
-        """ representation for debugging purposes """
-        return "%s(%r[:%d])" % (
-            self.__class__.__name__, self.builder, self.length)
-
-    def unwrap(self, space):
-        return self.force()
-
-    def str_w(self, space):
-        return self.force()
-
-    def buffer_w(self, space, flags):
-        return SimpleView(StringBuffer(self.force()))
-
-    def descr_len(self, space):
-        return space.newint(self.length)
-
-    def descr_add(self, space, w_other):
-        try:
-            other = W_BytesObject._op_val(space, w_other)
-        except OperationError as e:
-            if e.match(space, space.w_TypeError):
-                return space.w_NotImplemented
-            raise
-        if self.builder.getlength() != self.length:
-            builder = StringBuilder()
-            builder.append(self.force())
-        else:
-            builder = self.builder
-        builder.append(other)
-        return W_StringBufferObject(builder)
-
-    def descr_str(self, space):
-        # you cannot get subclasses of W_StringBufferObject here
-        assert type(self) is W_StringBufferObject
-        return self
-
-
-delegation_dict = {}
-for key, value in W_BytesObject.typedef.rawdict.iteritems():
-    if not isinstance(value, interp2app):
-        continue
-    if key in ('__len__', '__add__', '__str__'):
-        continue
-
-    func = value._code._bltin
-    args = inspect.getargs(func.func_code)
-    if args.varargs or args.keywords:
-        raise TypeError("Varargs and keywords not supported in unwrap_spec")
-    argspec = ', '.join([arg for arg in args.args[1:]])
-    func_code = py.code.Source("""
-    def f(self, %(args)s):
-        self.force()
-        return self.w_str.%(func_name)s(%(args)s)
-    """ % {'args': argspec, 'func_name': func.func_name})
-    d = {}
-    exec func_code.compile() in d
-    f = d['f']
-    f.func_defaults = func.func_defaults
-    f.__module__ = func.__module__
-    # necessary for unique identifiers for pickling
-    f.func_name = func.func_name
-    unwrap_spec_ = getattr(func, 'unwrap_spec', None)
-    if unwrap_spec_ is not None:
-        f = unwrap_spec(**unwrap_spec_)(f)
-    setattr(W_StringBufferObject, func.func_name, f)
-
-W_StringBufferObject.typedef = W_BytesObject.typedef
diff --git a/pypy/objspace/std/test/test_stdobjspace.py b/pypy/objspace/std/test/test_stdobjspace.py
--- a/pypy/objspace/std/test/test_stdobjspace.py
+++ b/pypy/objspace/std/test/test_stdobjspace.py
@@ -57,13 +57,6 @@
         cls = space._get_interplevel_cls(w_sequenceiterator)
         assert cls is W_AbstractSeqIterObject
 
-    def test_withstrbuf_fastpath_isinstance(self):
-        from pypy.objspace.std.bytesobject import W_AbstractBytesObject
-
-        space = gettestobjspace(withstrbuf=True)
-        cls = space._get_interplevel_cls(space.w_bytes)
-        assert cls is W_AbstractBytesObject
-
     def test_wrap_various_unsigned_types(self):
         import sys
         from rpython.rlib.rarithmetic import r_uint
diff --git a/pypy/objspace/std/test/test_strbufobject.py b/pypy/objspace/std/test/test_strbufobject.py
deleted file mode 100644
--- a/pypy/objspace/std/test/test_strbufobject.py
+++ /dev/null
@@ -1,85 +0,0 @@
-import py
-
-from pypy.objspace.std.test import test_bytesobject
-
-class AppTestStringObject(test_bytesobject.AppTestBytesObject):
-    spaceconfig = {"objspace.std.withstrbuf": True}
-
-    def test_basic(self):
-        import __pypy__
-        # cannot do "Hello, " + "World!" because cpy2.5 optimises this
-        # away on AST level
-        s = "Hello, ".__add__("World!")
-        assert type(s) is str
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-
-    def test_add_twice(self):
-        x = "a".__add__("b")
-        y = x + "c"
-        c = x + "d"
-        assert y == "abc"
-        assert c == "abd"
-
-    def test_add(self):
-        import __pypy__
-        all = ""
-        for i in range(20):
-            all += str(i)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(all)
-        assert all == "012345678910111213141516171819"
-
-    def test_hash(self):
-        import __pypy__
-        def join(s): return s[:len(s) // 2] + s[len(s) // 2:]
-        t = 'a' * 101
-        s = join(t)
-        assert 'W_StringBufferObject' in __pypy__.internal_repr(s)
-        assert hash(s) == hash(t)
-
-    def test_len(self):
-        s = "a".__add__("b")
-        r = "c".__add__("d")
-        t = s + r
-        assert len(s) == 2
-        assert len(r) == 2
-        assert len(t) == 4
-
-    def test_buffer(self):
-        s = b'a'.__add__(b'b')
-        assert buffer(s) == buffer(b'ab')
-        assert memoryview(s) == b'ab'
-
-    def test_add_strbuf(self):
-        # make three strbuf objects
-        s = 'a'.__add__('b')
-        t = 'x'.__add__('c')
-        u = 'y'.__add__('d')
-
-        # add two different strbufs to the same string
-        v = s + t
-        w = s + u
-
-        # check that insanity hasn't resulted.
-        assert v == "abxc"
-        assert w == "abyd"
-
-    def test_more_adding_fun(self):
-        s = 'a'.__add__('b') # s is a strbuf now
-        t = s + 'c'
-        u = s + 'd'
-        v = s + 'e'
-        assert v == 'abe'
-        assert u == 'abd'
-        assert t == 'abc'
-
-    def test_buh_even_more(self):
-        a = 'a'.__add__('b')
-        b = a + 'c'
-        c = '0'.__add__('1')
-        x = c + a
-        assert x == '01ab'
-
-    def test_add_non_string(self):
-        a = 'a'
-        a += 'b'
-        raises(TypeError, "a += 5")
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -514,13 +514,13 @@
     ne = eq
 
     def lt((tup1, tup2)):
-        raise Exception("unsupported: (...) < (...)")
+        raise AnnotatorError("unsupported: (...) < (...)")
     def le((tup1, tup2)):
-        raise Exception("unsupported: (...) <= (...)")
+        raise AnnotatorError("unsupported: (...) <= (...)")
     def gt((tup1, tup2)):
-        raise Exception("unsupported: (...) > (...)")
+        raise AnnotatorError("unsupported: (...) > (...)")
     def ge((tup1, tup2)):
-        raise Exception("unsupported: (...) >= (...)")
+        raise AnnotatorError("unsupported: (...) >= (...)")
 
 
 class __extend__(pairtype(SomeDict, SomeDict)):
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -15,7 +15,8 @@
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeException,
     SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked,
-    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty)
+    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty,
+    AnnotatorError)
 from rpython.annotator.classdesc import ClassDef, ClassDesc
 from rpython.annotator.listdef import ListDef, ListItem
 from rpython.annotator.dictdef import DictDef
@@ -343,7 +344,7 @@
         elif x is None:
             return s_None
         else:
-            raise Exception("Don't know how to represent %r" % (x,))
+            raise AnnotatorError("Don't know how to represent %r" % (x,))
         result.const = x
         return result
 
@@ -363,7 +364,7 @@
                 result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
-                    raise Exception("ClassDesc for object not supported")
+                    raise AnnotatorError("ClassDesc for object not supported")
                 if pyobj.__module__ == '__builtin__': # avoid making classdefs for builtin types
                     result = self.getfrozen(pyobj)
                 else:
@@ -400,7 +401,7 @@
                         msg = "object with a __call__ is not RPython"
                     else:
                         msg = "unexpected prebuilt constant"
-                    raise Exception("%s: %r" % (msg, pyobj))
+                    raise AnnotatorError("%s: %r" % (msg, pyobj))
                 result = self.getfrozen(pyobj)
             self.descs[obj_key] = result
             return result
@@ -589,7 +590,7 @@
         for name, value in dict.iteritems():
             if value is func:
                 return cls, name
-    raise Exception("could not match bound-method to attribute name: %r" % (boundmeth,))
+    raise AnnotatorError("could not match bound-method to attribute name: %r" % (boundmeth,))
 
 def ishashable(x):
     try:
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -550,8 +550,8 @@
                                         "with _mixin_: %r" % (cls,))
                 base = b1
         if mixins_before and mixins_after:
-            raise Exception("unsupported: class %r has mixin bases both"
-                            " before and after the regular base" % (self,))
+            raise AnnotatorError("unsupported: class %r has mixin bases both"
+                                 " before and after the regular base" % (self,))
         self.add_mixins(mixins_after, check_not_in=base)
         self.add_mixins(mixins_before)
         self.add_sources_for_class(cls)
@@ -569,8 +569,8 @@
                 attrs.update(decl)
             if self.basedesc is not None:
                 if self.basedesc.all_enforced_attrs is None:
-                    raise Exception("%r has slots or _attrs_, "
-                                    "but not its base class" % (cls,))
+                    raise AnnotatorError("%r has slots or _attrs_, "
+                                         "but not its base class" % (cls,))
                 attrs.update(self.basedesc.all_enforced_attrs)
             self.all_enforced_attrs = attrs
 
@@ -714,8 +714,8 @@
                 try:
                     args.fixedunpack(0)
                 except ValueError:
-                    raise Exception("default __init__ takes no argument"
-                                    " (class %s)" % (self.name,))
+                    raise AnnotatorError("default __init__ takes no argument"
+                                         " (class %s)" % (self.name,))
             elif self.pyobj is Exception:
                 # check explicitly against "raise Exception, x" where x
                 # is a low-level exception pointer
@@ -824,8 +824,8 @@
         # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist,
         # test_immutable_list_out_of_instance.
         if self._detect_invalid_attrs and attr in self._detect_invalid_attrs:
-            raise Exception("field %r was migrated to %r from a subclass in "
-                            "which it was declared as _immutable_fields_" %
+            raise AnnotatorError("field %r was migrated to %r from a subclass in "
+                                 "which it was declared as _immutable_fields_" %
                             (attr, self.pyobj))
         search1 = '%s[*]' % (attr,)
         search2 = '%s?[*]' % (attr,)
@@ -858,7 +858,7 @@
             # call to a single class, look at the result annotation
             # in case it was specialized
             if not isinstance(s_result, SomeInstance):
-                raise Exception("calling a class didn't return an instance??")
+                raise AnnotatorError("calling a class didn't return an instance??")
             classdefs = [s_result.classdef]
         else:
             # call to multiple classes: specialization not supported
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -314,8 +314,8 @@
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
-            raise Exception("%r: signature and enforceargs cannot both be "
-                            "used" % (self,))
+            raise AnnotatorError("%r: signature and enforceargs cannot both be "
+                                 "used" % (self,))
         if enforceargs:
             if not callable(enforceargs):
                 from rpython.annotator.signature import Sig
@@ -432,7 +432,7 @@
     def func_args(self, args):
         from rpython.annotator.model import SomeInstance
         if self.selfclassdef is None:
-            raise Exception("calling %r" % (self,))
+            raise AnnotatorError("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4584,7 +4584,7 @@
             return Ellipsis
         a = self.RPythonAnnotator()
         e = py.test.raises(Exception, a.build_types, f, [])
-        assert str(e.value) == "Don't know how to represent Ellipsis"
+        assert "Don't know how to represent Ellipsis" in str(e.value)
 
     def test_must_be_light_finalizer(self):
         from rpython.rlib import rgc
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -185,8 +185,8 @@
         return SomeString()
 
     def id(self):
-        raise Exception("cannot use id() in RPython; "
-                        "see objectmodel.compute_xxx()")
+        raise AnnotatorError("cannot use id() in RPython; "
+                             "see objectmodel.compute_xxx()")
 
     def int(self):
         return SomeInteger()
@@ -421,7 +421,7 @@
     def setslice(self, s_start, s_stop, s_iterable):
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
-            raise Exception("list[start:stop] = x: x must be a list")
+            raise AnnotatorError("list[start:stop] = x: x must be a list")
         self.listdef.mutate()
         self.listdef.agree(getbookkeeper(), s_iterable.listdef)
         self.listdef.resize()
diff --git a/rpython/translator/platform/openbsd.py b/rpython/translator/platform/openbsd.py
--- a/rpython/translator/platform/openbsd.py
+++ b/rpython/translator/platform/openbsd.py
@@ -16,5 +16,14 @@
         libraries=set(libraries + ("intl", "iconv"))
         return ['-l%s' % lib for lib in libraries if lib not in ["crypt", "dl", "rt"]]
 
+    def makefile_link_flags(self):
+        # On OpenBSD, we need to build the final binary with the link flags
+        # below. However, if we modify self.link_flags to include these, the
+        # various platform check binaries that RPython builds end up with these
+        # flags: since these binaries are generally located on /tmp -- which
+        # isn't a wxallowed file system -- that gives rise to "permission
+        # denied" errors, which kill the build.
+        return list(self.link_flags) + ["-Wl,-z,wxneeded"]
+
 class OpenBSD_64(OpenBSD):
     shared_only = ('-fPIC',)
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -98,6 +98,9 @@
     def get_shared_only_compile_flags(self):
         return tuple(self.shared_only) + ('-fvisibility=hidden',)
 
+    def makefile_link_flags(self):
+        return list(self.link_flags)
+
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
                      no_precompile_cfiles = [], config=None):
@@ -113,7 +116,7 @@
         else:
             exe_name = exe_name.new(ext=self.exe_ext)
 
-        linkflags = list(self.link_flags)
+        linkflags = self.makefile_link_flags()
         if shared:
             linkflags = self._args_for_shared(linkflags)
 

From pypy.commits at gmail.com  Tue May 23 04:50:15 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 01:50:15 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: this this test_pypy_c test:
 now struct.unpack generates a fast raw_load_i;
 use 'i' instead of '<i' to make sure that we get the same code even on big
 endian machines
Message-ID: <5923f7c7.493a1c0a.3a5fb.139e@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91383:6878ac6c89b8
Date: 2017-05-23 10:49 +0200
http://bitbucket.org/pypy/pypy/changeset/6878ac6c89b8/

Log:	this this test_pypy_c test: now struct.unpack generates a fast
	raw_load_i; use 'i' instead of '<i' to make sure that we get the
	same code even on big endian machines

diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -34,29 +34,13 @@
             i = 0
             while i < n:
                 i += 1
-                struct.unpack('<i', a)  # ID: unpack
+                struct.unpack('i', a)  # ID: unpack
             return i
         log = self.run(main, [1000])
         assert log.result == 1000
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('unpack', """
             guard_not_invalidated(descr=...)
-            p90 = newstr(4)
-            call_n(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=<Callv 0 irii EF=5>)
-            guard_no_exception(descr=...)
-            i91 = strgetitem(p90, 0)
-            i92 = strgetitem(p90, 1)
-            i93 = int_lshift(i92, 8)
-            i94 = int_or(i91, i93)
-            i95 = strgetitem(p90, 2)
-            i96 = int_lshift(i95, 16)
-            i97 = int_or(i94, i96)
-            i98 = strgetitem(p90, 3)
-            i99 = int_ge(i98, 128)
-            guard_false(i99, descr=...)
-            i100 = int_lshift(i98, 24)
-            i101 = int_or(i97, i100)
-            i102 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value 0>)
-            i103 = int_lt(i102, 0)
-            guard_false(i103, descr=...)
+            i66 = raw_load_i(i53, 0, descr=<ArrayS 4>)
+            --TICK--
         """)

From pypy.commits at gmail.com  Tue May 23 04:58:20 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 01:58:20 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: make this test
 endianess-independent: hopefully this fixed the failure on S390X
Message-ID: <5923f9ac.09071c0a.a30f2.0e70@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91384:18131865f52d
Date: 2017-05-23 10:57 +0200
http://bitbucket.org/pypy/pypy/changeset/18131865f52d/

Log:	make this test endianess-independent: hopefully this fixed the
	failure on S390X

diff --git a/rpython/rlib/test/test_buffer.py b/rpython/rlib/test/test_buffer.py
--- a/rpython/rlib/test/test_buffer.py
+++ b/rpython/rlib/test/test_buffer.py
@@ -140,9 +140,10 @@
 class TestRawBufferTypedWrite(object):
 
     def test_typed_write(self):
+        expected = struct.pack('=H', 0xABCD) + '\xff'*6
         buf = MyRawBuffer('\xff' * 8, readonly=False)
         buf.typed_write(rffi.USHORT, 0, 0xABCD)
-        assert buf.as_str() == '\xcd\xab\xff\xff\xff\xff\xff\xff'
+        assert buf.as_str() == expected
         assert buf.typed_read(rffi.USHORT, 0) == 0xABCD
     
 

From pypy.commits at gmail.com  Tue May 23 07:42:51 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 04:42:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test_whatsnew
Message-ID: <5924203b.d2991c0a.f9880.4ca4@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91385:ed256c4d2705
Date: 2017-05-23 13:38 +0200
http://bitbucket.org/pypy/pypy/changeset/ed256c4d2705/

Log:	fix test_whatsnew

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -56,3 +56,6 @@
 
 Remove faulty fastpath from ctypes
 
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt

From pypy.commits at gmail.com  Tue May 23 07:42:53 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 04:42:53 -0700 (PDT)
Subject: [pypy-commit] pypy default: kill this test: originally it was meant
 to check that we take the fastpath when doing a ctypes call,
 but now the fastpath has gone and thus there is nothing to test. Moreover,
 most of the logic has been skipped since forever, apparently
Message-ID: <5924203d.229adf0a.eb186.4f11@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91386:a8f0d06d39a8
Date: 2017-05-23 13:42 +0200
http://bitbucket.org/pypy/pypy/changeset/a8f0d06d39a8/

Log:	kill this test: originally it was meant to check that we take the
	fastpath when doing a ctypes call, but now the fastpath has gone and
	thus there is nothing to test. Moreover, most of the logic has been
	skipped since forever, apparently

diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -110,38 +110,6 @@
         loops = log.loops_by_id('sleep')
         assert len(loops) == 1 # make sure that we actually JITted the loop
 
-    def test_ctypes_call(self):
-        from rpython.rlib.test.test_clibffi import get_libm_name
-        def main(libm_name):
-            import ctypes
-            libm = ctypes.CDLL(libm_name)
-            fabs = libm.fabs
-            fabs.argtypes = [ctypes.c_double]
-            fabs.restype = ctypes.c_double
-            x = -4
-            i = 0
-            while i < 300:
-                x = fabs(x)
-                x = x - 100
-                i += 1
-            return fabs._ptr.getaddr(), x
-
-        libm_name = get_libm_name(sys.platform)
-        log = self.run(main, [libm_name], import_site=True)
-        fabs_addr, res = log.result
-        assert res == -4.0
-        loop, = log.loops_by_filename(self.filepath)
-        ops = loop.allops()
-        opnames = log.opnames(ops)
-        assert opnames.count('new_with_vtable') == 1 # only the virtualref
-        py.test.skip("XXX re-optimize _ffi for the JIT?")
-        assert opnames.count('call_release_gil') == 1
-        idx = opnames.index('call_release_gil')
-        call = ops[idx]
-        assert (call.args[0] == 'ConstClass(fabs)' or    # e.g. OS/X
-                int(call.args[0]) == fabs_addr)
-
-
     def test__ffi_struct(self):
         def main():
             from _rawffi.alt import _StructDescr, Field, types

From pypy.commits at gmail.com  Tue May 23 08:43:23 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:43:23 -0700 (PDT)
Subject: [pypy-commit] pypy better-test-whatsnew: improve this test and make
 sure that the CURRENT branch is documented: this way we get a failure early
 before merging the branch instead of having test_whatsnew failing on
 default the day after
Message-ID: <59242e6b.61a9df0a.cd380.5f6d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: better-test-whatsnew
Changeset: r91388:ae2ab7014840
Date: 2017-05-23 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/ae2ab7014840/

Log:	improve this test and make sure that the CURRENT branch is
	documented: this way we get a failure early before merging the
	branch instead of having test_whatsnew failing on default the day
	after

diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -101,6 +101,8 @@
     assert not not_documented
     if branch == 'default':
         assert not not_merged
+    else:
+        assert branch in documented, 'Please document this branch before merging: %s' % branch
 
 def test_startrev_on_default():
     doc = ROOT.join('pypy', 'doc')

From pypy.commits at gmail.com  Tue May 23 08:43:26 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:43:26 -0700 (PDT)
Subject: [pypy-commit] pypy better-test-whatsnew: close branch about to be
 merged
Message-ID: <59242e6e.748fdf0a.80d8b.6463@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: better-test-whatsnew
Changeset: r91390:57efe071e53a
Date: 2017-05-23 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/57efe071e53a/

Log:	close branch about to be merged


From pypy.commits at gmail.com  Tue May 23 08:43:28 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:43:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge the branch better-test-whatsnew:
 now you must document your branch in whatsnew-HEAD.rst BEFORE merging it
Message-ID: <59242e70.05421c0a.62f92.65bc@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91391:b7531baf47cf
Date: 2017-05-23 14:42 +0200
http://bitbucket.org/pypy/pypy/changeset/b7531baf47cf/

Log:	merge the branch better-test-whatsnew: now you must document your
	branch in whatsnew-HEAD.rst BEFORE merging it

diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -101,6 +101,8 @@
     assert not not_documented
     if branch == 'default':
         assert not not_merged
+    else:
+        assert branch in documented, 'Please document this branch before merging: %s' % branch
 
 def test_startrev_on_default():
     doc = ROOT.join('pypy', 'doc')
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -59,3 +59,5 @@
 .. branch: sockopt_zero
 
 Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew

From pypy.commits at gmail.com  Tue May 23 08:43:21 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:43:21 -0700 (PDT)
Subject: [pypy-commit] pypy better-test-whatsnew: a branch where to improve
 test_whatsnew:
Message-ID: <59242e69.1b8bdf0a.76b29.308d@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: better-test-whatsnew
Changeset: r91387:c96a110a73d2
Date: 2017-05-23 14:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c96a110a73d2/

Log:	a branch where to improve test_whatsnew:


From pypy.commits at gmail.com  Tue May 23 08:43:24 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:43:24 -0700 (PDT)
Subject: [pypy-commit] pypy better-test-whatsnew: add this branch to the
 documented ones, and make the test passing again
Message-ID: <59242e6c.44d21c0a.e1060.6a72@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: better-test-whatsnew
Changeset: r91389:95360dc76049
Date: 2017-05-23 14:41 +0200
http://bitbucket.org/pypy/pypy/changeset/95360dc76049/

Log:	add this branch to the documented ones, and make the test passing
	again

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -59,3 +59,5 @@
 .. branch: sockopt_zero
 
 Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew

From pypy.commits at gmail.com  Tue May 23 08:51:14 2017
From: pypy.commits at gmail.com (antocuni)
Date: Tue, 23 May 2017 05:51:14 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: remove dead import (thanks
 cfbolz)
Message-ID: <59243042.ce1a1c0a.36e46.5e42@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91392:8dcd7b31db1e
Date: 2017-05-23 14:50 +0200
http://bitbucket.org/pypy/pypy/changeset/8dcd7b31db1e/

Log:	remove dead import (thanks cfbolz)

diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -1,5 +1,4 @@
 from __future__ import with_statement
-from rpython.annotator import model as annmodel
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype

From pypy.commits at gmail.com  Wed May 24 08:02:50 2017
From: pypy.commits at gmail.com (Kounavi)
Date: Wed, 24 May 2017 05:02:50 -0700 (PDT)
Subject: [pypy-commit] pypy
 Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235: Fix typo
 (depricate to deprecate) :P
Message-ID: <5925766a.0a5e1c0a.91314.d663@mx.google.com>

Author: Iraklis D. <sithiraklis at yahoo.gr>
Branch: Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235
Changeset: r91393:9fe360fea796
Date: 2017-05-24 11:16 +0000
http://bitbucket.org/pypy/pypy/changeset/9fe360fea796/

Log:	Fix typo (depricate to deprecate) :P

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -23,7 +23,7 @@
 Installing Visual Compiler v9 (for Python 2.7)
 ----------------------------------------------
 
-This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
 was checked in Nov 2016). Note that the compiler suite will be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.

From pypy.commits at gmail.com  Wed May 24 08:02:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 24 May 2017 05:02:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 Kounavi/pypy/Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235 (pull
 request #550)
Message-ID: <5925766c.09631c0a.d9202.4e9f@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r91394:62e05f75166d
Date: 2017-05-24 12:02 +0000
http://bitbucket.org/pypy/pypy/changeset/62e05f75166d/

Log:	Merged in Kounavi/pypy/Kounavi/fix-typo-depricate-to-
	deprecate-p-1495624547235 (pull request #550)

	Fix typo (depricate to deprecate) :P

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -23,7 +23,7 @@
 Installing Visual Compiler v9 (for Python 2.7)
 ----------------------------------------------
 
-This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
 was checked in Nov 2016). Note that the compiler suite will be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.

From pypy.commits at gmail.com  Wed May 24 11:28:35 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 24 May 2017 08:28:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: document the merged branch
Message-ID: <5925a6a3.f7abdf0a.cd41e.7054@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91397:add017ddb1bc
Date: 2017-05-24 17:27 +0200
http://bitbucket.org/pypy/pypy/changeset/add017ddb1bc/

Log:	document the merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -61,3 +61,10 @@
 Passing a buffersize of 0 to socket.getsockopt
 
 .. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings

From pypy.commits at gmail.com  Wed May 24 11:28:30 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 24 May 2017 08:28:30 -0700 (PDT)
Subject: [pypy-commit] pypy faster-rstruct-2: close branch about to be merged
Message-ID: <5925a69e.505c1c0a.a90fb.d828@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: faster-rstruct-2
Changeset: r91395:43a0bd03dd95
Date: 2017-05-24 17:24 +0200
http://bitbucket.org/pypy/pypy/changeset/43a0bd03dd95/

Log:	close branch about to be merged


From pypy.commits at gmail.com  Wed May 24 11:28:33 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 24 May 2017 08:28:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge the faster-rstruct-2 branch: this
 makes struct.pack and struct.pack_into much faster by using raw_store or
 gc_store_indexed whenever possible. Moreover,
 it enables the existing struct.unpack fast path to all the existing buffer
 types, whereas previously it was enabled only for strings
Message-ID: <5925a6a1.89aadf0a.b46c0.ec19@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91396:d3895494491b
Date: 2017-05-24 17:26 +0200
http://bitbucket.org/pypy/pypy/changeset/d3895494491b/

Log:	merge the faster-rstruct-2 branch: this makes struct.pack and
	struct.pack_into much faster by using raw_store or gc_store_indexed
	whenever possible. Moreover, it enables the existing struct.unpack
	fast path to all the existing buffer types, whereas previously it
	was enabled only for strings

diff too long, truncating to 2000 out of 3450 lines

diff --git a/pypy/module/__pypy__/bytebuffer.py b/pypy/module/__pypy__/bytebuffer.py
--- a/pypy/module/__pypy__/bytebuffer.py
+++ b/pypy/module/__pypy__/bytebuffer.py
@@ -2,29 +2,8 @@
 # A convenient read-write buffer.  Located here for want of a better place.
 #
 
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import ByteBuffer
 from pypy.interpreter.gateway import unwrap_spec
-from rpython.rlib.rgc import nonmoving_raw_ptr_for_resizable_list
-
-
-class ByteBuffer(Buffer):
-    _immutable_ = True
-
-    def __init__(self, len):
-        self.data = ['\x00'] * len
-        self.readonly = False
-
-    def getlength(self):
-        return len(self.data)
-
-    def getitem(self, index):
-        return self.data[index]
-
-    def setitem(self, index, char):
-        self.data[index] = char
-
-    def get_raw_address(self):
-        return nonmoving_raw_ptr_for_resizable_list(self.data)
 
 @unwrap_spec(length=int)
 def bytebuffer(space, length):
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -6,13 +6,13 @@
 from pypy.objspace.std.bufferobject import W_Buffer
 from pypy.interpreter.buffer import SimpleView
 
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(Buffer):
+class LLBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
@@ -35,7 +35,7 @@
     def getslice(self, start, stop, step, size):
         if step == 1:
             return rffi.charpsize2str(rffi.ptradd(self.raw_cdata, start), size)
-        return Buffer.getslice(self, start, stop, step, size)
+        return RawBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         raw_cdata = rffi.ptradd(self.raw_cdata, start)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,11 +1,10 @@
 from rpython.rtyper.lltypesystem import rffi
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,5 @@
 from rpython.rlib import jit, rgc
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
@@ -796,7 +796,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayBuffer(Buffer):
+class ArrayBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, array, readonly):
@@ -840,7 +840,7 @@
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
                 self.array._charbuf_stop()
-        return Buffer.getslice(self, start, stop, step, size)
+        return RawBuffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
         return self.array._charbuf_start()
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -552,6 +552,7 @@
         to compress, call the flush() method to finish the compression process,
         and return what is left in the internal buffers."""
 
+        assert data is not None
         datasize = len(data)
 
         if datasize == 0:
@@ -662,6 +663,7 @@
         was found after the end of stream, it'll be ignored and saved in
         unused_data attribute."""
 
+        assert data is not None
         if not self.running:
             raise oefmt(self.space.w_EOFError,
                         "end of stream was already found")
@@ -715,6 +717,7 @@
     use an instance of BZ2Compressor instead. The compresslevel parameter, if
     given, must be a number between 1 and 9."""
 
+    assert data is not None
     if compresslevel < 1 or compresslevel > 9:
         raise oefmt(space.w_ValueError,
                     "compresslevel must be between 1 and 9")
@@ -757,6 +760,7 @@
     Decompress data in one shot. If you want to decompress data sequentially,
     use an instance of BZ2Decompressor instead."""
 
+    assert data is not None
     in_bufsize = len(data)
     if in_bufsize == 0:
         return space.newbytes("")
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -22,8 +22,7 @@
 from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
 from rpython.tool.sourcetools import func_renamer
@@ -427,7 +426,7 @@
 fq = FQ()
 
 
-class CBuffer(Buffer):
+class CBuffer(RawBuffer):
     _immutable_ = True
     def __init__(self, view):
         self.view = view
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -3,7 +3,7 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -702,7 +702,8 @@
     def __del__(self):
         free_raw_storage(self.storage)
 
-class ArrayData(Buffer):
+
+class ArrayData(RawBuffer):
     _immutable_ = True
     def __init__(self, impl, readonly):
         self.impl = impl
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -3,7 +3,7 @@
 from pypy.interpreter.typedef import TypeDef
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from rpython.rlib import rmmap, rarithmetic, objectmodel
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
 
@@ -330,7 +330,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(Buffer):
+class MMapBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):
@@ -350,7 +350,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return Buffer.getslice(self, start, stop, step, size)
+            return RawBuffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -34,29 +34,13 @@
             i = 0
             while i < n:
                 i += 1
-                struct.unpack('<i', a)  # ID: unpack
+                struct.unpack('i', a)  # ID: unpack
             return i
         log = self.run(main, [1000])
         assert log.result == 1000
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('unpack', """
             guard_not_invalidated(descr=...)
-            p90 = newstr(4)
-            call_n(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=<Callv 0 irii EF=5>)
-            guard_no_exception(descr=...)
-            i91 = strgetitem(p90, 0)
-            i92 = strgetitem(p90, 1)
-            i93 = int_lshift(i92, 8)
-            i94 = int_or(i91, i93)
-            i95 = strgetitem(p90, 2)
-            i96 = int_lshift(i95, 16)
-            i97 = int_or(i94, i96)
-            i98 = strgetitem(p90, 3)
-            i99 = int_ge(i98, 128)
-            guard_false(i99, descr=...)
-            i100 = int_lshift(i98, 24)
-            i101 = int_or(i97, i100)
-            i102 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value 0>)
-            i103 = int_lt(i102, 0)
-            guard_false(i103, descr=...)
+            i66 = raw_load_i(i53, 0, descr=<ArrayS 4>)
+            --TICK--
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -30,32 +30,33 @@
         loop, = log.loops_by_filename(self.filepath)
         # This could, of course stand some improvement, to remove all these
         # arithmatic ops, but we've removed all the core overhead.
-        assert loop.match_by_id("pack", """
-            guard_not_invalidated(descr=...)
-            # struct.pack
-            %s
-            i11 = int_and(i4, 255)
-            i13 = int_rshift(i4, 8)
-            i14 = int_and(i13, 255)
-            i16 = int_rshift(i13, 8)
-            i17 = int_and(i16, 255)
-            i19 = int_rshift(i16, 8)
-            i20 = int_and(i19, 255)
-        """ % extra)
+        if sys.byteorder == 'little':
+            # on little endian machines, we take the fast path and store the
+            # value using gc_store_indexed
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                p75 = newstr(4)
+                gc_store_indexed(p75, 0, _, 1, _, 4, descr=...)
+            """ % extra)
+        else:
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                i11 = int_and(i4, 255)
+                i13 = int_rshift(i4, 8)
+                i14 = int_and(i13, 255)
+                i16 = int_rshift(i13, 8)
+                i17 = int_and(i16, 255)
+                i19 = int_rshift(i16, 8)
+                i20 = int_and(i19, 255)
+            """ % extra)
 
         if sys.byteorder == 'little':
-            # the newstr and the strsetitems are because the string is forced,
-            # which is in turn because the optimizer doesn't know how to handle a
-            # gc_load_indexed_i on a virtual string. It could be improved, but it
-            # is also true that in real life cases struct.unpack is called on
-            # strings which come from the outside, so it's a minor issue.
             assert loop.match_by_id("unpack", """
                 # struct.unpack
-                p88 = newstr(4)
-                strsetitem(p88, 0, i11)
-                strsetitem(p88, 1, i14)
-                strsetitem(p88, 2, i17)
-                strsetitem(p88, 3, i20)
                 i91 = gc_load_indexed_i(p88, 0, 1, _, -4)
             """)
         else:
@@ -93,27 +94,106 @@
             assert loop.match_by_id('pack', """
                 guard_not_invalidated(descr=...)
                 # struct.pack
+                p85 = newstr(8)
+                gc_store_indexed(p85, 0, -1, 1, _, 4, descr=...)
                 %s
-                i11 = int_and(i4, 255)
-                i13 = int_rshift(i4, 8)
-                i14 = int_and(i13, 255)
-                i16 = int_rshift(i13, 8)
-                i17 = int_and(i16, 255)
-                i19 = int_rshift(i16, 8)
-                i20 = int_and(i19, 255)
+                gc_store_indexed(p85, 4, _, 1, _, 4, descr=...)
             """ % extra)
 
             assert loop.match_by_id('unpack', """
                 # struct.unpack
-                p88 = newstr(8)
-                strsetitem(p88, 0, 255)
-                strsetitem(p88, 1, 255)
-                strsetitem(p88, 2, 255)
-                strsetitem(p88, 3, 255)
-                strsetitem(p88, 4, i11)
-                strsetitem(p88, 5, i14)
-                strsetitem(p88, 6, i17)
-                strsetitem(p88, 7, i20)
                 i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
                 i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
             """)
+
+    def test_unpack_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = array.array('b', buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i65 = raw_load_i(i49, 0, descr=<ArrayS 2>)
+        """)
+
+    def test_unpack_bytearray(self):
+        def main(n):
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = bytearray(buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        # the offset of gc_load_indexed_i used to be the constant 0. However,
+        # now it is 'i46' because we need to add 0 to
+        # W_BytearrayObject._offset
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i70 = gc_load_indexed_i(p48, i46, 1, _, -2)
+        """)
+
+    def test_pack_into_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = array.array('b', '\x00'*8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            i65 = int_le(i58, 32767)
+            guard_true(i65, descr=...)
+            raw_store(i55, 4, i58, descr=<ArrayS 2>)
+        """)
+
+    def test_pack_into_bytearray(self):
+        def main(n):
+            import struct
+            buf = bytearray(8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            p68 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__data \d+>)
+            i69 = getfield_gc_i(p68, descr=<FieldS list.length \d+>)
+            i70 = getfield_gc_i(p14, descr=<FieldS pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__offset \d+>)
+            i71 = int_sub(i69, i70)
+            i73 = int_sub(i71, 4)
+            i75 = int_lt(i73, 2)
+            guard_false(i75, descr=...)
+            i77 = int_le(i62, 32767)
+            guard_true(i77, descr=...)
+            p78 = getfield_gc_r(p68, descr=<FieldP list.items \d+>)
+            i81 = int_add(4, i70)
+            gc_store_indexed(p78, i81, i62, 1, _, 2, descr=<ArrayS 2>)
+        """)
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -2,7 +2,6 @@
                                       maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 
@@ -10,11 +9,15 @@
 
 
 class PackFormatIterator(FormatIterator):
-    def __init__(self, space, args_w, size):
+    def __init__(self, space, wbuf, args_w):
         self.space = space
         self.args_w = args_w
         self.args_index = 0
-        self.result = StringBuilder(size)
+        self.pos = 0
+        self.wbuf = wbuf
+
+    def advance(self, count):
+        self.pos += count
 
     # This *should* be always unroll safe, the only way to get here is by
     # unroll the interpret function, which means the fmt is const, and thus
@@ -31,8 +34,10 @@
 
     @jit.unroll_safe
     def align(self, mask):
-        pad = (-self.result.getlength()) & mask
-        self.result.append_multiple_char('\x00', pad)
+        pad = (-self.pos) & mask
+        for i in range(self.pos, self.pos+pad):
+            self.wbuf.setitem(i, '\x00')
+        self.advance(pad)
 
     def finished(self):
         if self.args_index != len(self.args_w):
@@ -140,13 +145,20 @@
         if self.pos != self.length:
             raise StructError("unpack str size too long for format")
 
+    def can_advance(self, count):
+        end = self.pos + count
+        return end <= self.length
+
+    def advance(self, count):
+        if not self.can_advance(count):
+            raise StructError("unpack str size too short for format")
+        self.pos += count
+
     def read(self, count):
-        end = self.pos + count
-        if end > self.length:
-            raise StructError("unpack str size too short for format")
-        s = self.buf.getslice(self.pos, end, 1, count)
-        self.pos = end
-        return s
+        curpos = self.pos
+        end = curpos + count
+        self.advance(count) # raise if we are out of bound
+        return self.buf.getslice(curpos, end, 1, count)
 
     @specialize.argtype(1)
     def appendobj(self, value):
@@ -182,9 +194,8 @@
     def get_pos(self):
         return self.pos
 
-    def get_buffer_as_string_maybe(self):
-        string, pos = self.buf.as_str_and_offset_maybe()
-        return string, pos+self.pos
+    def get_buffer_and_pos(self):
+        return self.buf, self.pos
 
     def skip(self, size):
         self.read(size) # XXX, could avoid taking the slice
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,5 +1,6 @@
 from rpython.rlib import jit
 from rpython.rlib.buffer import SubBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
@@ -40,18 +41,17 @@
 
 def _pack(space, format, args_w):
     """Return string containing values v1, v2, ... packed according to fmt."""
-    if jit.isconstant(format):
-        size = _calcsize(space, format)
-    else:
-        size = 8
-    fmtiter = PackFormatIterator(space, args_w, size)
+    size = _calcsize(space, format)
+    wbuf = MutableStringBuffer(size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
         raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
         raise OperationError(get_error(space), space.newtext(e.msg))
-    return fmtiter.result.build()
+    assert fmtiter.pos == wbuf.getlength(), 'missing .advance() or wrong calcsize()'
+    return wbuf.finish()
 
 
 @unwrap_spec(format='text')
@@ -59,22 +59,28 @@
     return space.newbytes(_pack(space, format, args_w))
 
 
-# XXX inefficient
 @unwrap_spec(format='text', offset=int)
 def pack_into(space, format, w_buffer, offset, args_w):
     """ Pack the values v1, v2, ... according to fmt.
 Write the packed bytes into the writable buffer buf starting at offset
     """
-    res = _pack(space, format, args_w)
+    size = _calcsize(space, format)
     buf = space.getarg_w('w*', w_buffer)
     if offset < 0:
         offset += buf.getlength()
-    size = len(res)
     if offset < 0 or (buf.getlength() - offset) < size:
         raise oefmt(get_error(space),
                     "pack_into requires a buffer of at least %d bytes",
                     size)
-    buf.setslice(offset, res)
+    #
+    wbuf = SubBuffer(buf, offset, size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
+    try:
+        fmtiter.interpret(format)
+    except StructOverflowError as e:
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
+    except StructError as e:
+        raise OperationError(get_error(space), space.newtext(e.msg))
 
 
 def _unpack(space, format, buf):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -491,7 +491,7 @@
 
 
 class AppTestFastPath(object):
-    spaceconfig = dict(usemodules=['struct', '__pypy__'])
+    spaceconfig = dict(usemodules=['array', 'struct', '__pypy__'])
 
     def setup_class(cls):
         from rpython.rlib.rstruct import standardfmttable
@@ -510,7 +510,43 @@
         from rpython.rlib.rstruct import standardfmttable
         standardfmttable.ALLOW_SLOWPATH = True
 
+    def test_unpack_simple(self):
+        buf = self.struct.pack("iii", 0, 42, 43)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
     def test_unpack_from(self):
         buf = self.struct.pack("iii", 0, 42, 43)
         offset = self.struct.calcsize("i")
         assert self.struct.unpack_from("ii", buf, offset) == (42, 43)
+
+    def test_unpack_bytearray(self):
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = bytearray(data)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_unpack_array(self):
+        import array
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = array.array('c', data)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_pack_into_bytearray(self):
+        expected = self.struct.pack("ii", 42, 43)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("ii", buf, 0, 42, 43)
+        assert buf == expected
+
+    def test_pack_into_bytearray_padding(self):
+        expected = self.struct.pack("xxi", 42)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("xxi", buf, 0, 42)
+        assert buf == expected
+
+    def test_pack_into_bytearray_delete(self):
+        expected = self.struct.pack("i", 42)
+        # force W_BytearrayObject._delete_from_start
+        buf = bytearray(64)
+        del buf[:8]
+        self.struct.pack_into("i", buf, 0, 42)
+        buf = buf[:len(expected)]
+        assert buf == expected
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -6,10 +6,11 @@
 from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-        nonmoving_raw_ptr_for_resizable_list)
+                              nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import Buffer
-
+from rpython.rlib.buffer import (GCBuffer,
+                                 get_gc_data_for_list_of_chars,
+                                 get_gc_data_offset_for_list_of_chars)
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import WrappedDefault, interp2app, unwrap_spec
@@ -1257,7 +1258,8 @@
         start += step
 
 
-class BytearrayBuffer(Buffer):
+ at GCBuffer.decorate
+class BytearrayBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, ba, readonly=False):
@@ -1287,7 +1289,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return GCBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
@@ -1302,6 +1304,16 @@
         p = rffi.ptradd(p, ba._offset)
         return p
 
+    @staticmethod
+    def _get_gc_data_offset():
+        return get_gc_data_offset_for_list_of_chars()
+
+    def _get_gc_data_extra_offset(self):
+        return self.ba._offset
+
+    def _get_gc_data(self):
+        return get_gc_data_for_list_of_chars(self.ba._data)
+
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -1,5 +1,5 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint
-from rpython.rlib.rstring import StringBuilder
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
 
@@ -190,9 +190,9 @@
 
 
 def pack_float(f):
-    result = StringBuilder(8)
-    ieee.pack_float(result, f, 8, False)
-    return result.build()
+    buf = MutableStringBuffer(8)
+    ieee.pack_float(buf, 0, f, 8, False)
+    return buf.finish()
 
 def unpack_float(s):
     return ieee.unpack_float(s, False)
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -4,6 +4,7 @@
     compute_hash, compute_unique_id, import_from_mixin,
     enforceargs)
 from rpython.rlib.buffer import StringBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstring import StringBuilder, UnicodeBuilder
 from rpython.rlib.runicode import (
     make_unicode_escape_function, str_decode_ascii, str_decode_utf_8,
@@ -84,10 +85,12 @@
 
     def readbuf_w(self, space):
         from rpython.rlib.rstruct.unichar import pack_unichar, UNICODE_SIZE
-        builder = StringBuilder(len(self._value) * UNICODE_SIZE)
+        buf = MutableStringBuffer(len(self._value) * UNICODE_SIZE)
+        pos = 0
         for unich in self._value:
-            pack_unichar(unich, builder)
-        return StringBuffer(builder.build())
+            pack_unichar(unich, buf, pos)
+            pos += UNICODE_SIZE
+        return StringBuffer(buf.finish())
 
     def writebuf_w(self, space):
         raise oefmt(space.w_TypeError,
diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -0,0 +1,9 @@
+from rpython.jit.backend.arm.test.support import JitARMMixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(JitARMMixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+    pass
+
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -716,16 +716,28 @@
         else:
             return self.bh_raw_load_i(struct, offset, descr)
 
+    def _get_int_type_from_size(self, size):
+        if   size == 1:
+            return rffi.UCHAR
+        elif size == 2:
+            return rffi.USHORT
+        elif size == 4:
+            return rffi.UINT
+        elif size == 8:
+            return rffi.ULONGLONG
+        elif size == -1:
+            return rffi.SIGNEDCHAR
+        elif size == -2:
+            return rffi.SHORT
+        elif size == -4:
+            return rffi.INT
+        elif size == -8:
+            return rffi.LONGLONG
+        else:
+            raise NotImplementedError(size)
+    
     def bh_gc_load_indexed_i(self, struct, index, scale, base_ofs, bytes):
-        if   bytes == 1: T = rffi.UCHAR
-        elif bytes == 2: T = rffi.USHORT
-        elif bytes == 4: T = rffi.UINT
-        elif bytes == 8: T = rffi.ULONGLONG
-        elif bytes == -1: T = rffi.SIGNEDCHAR
-        elif bytes == -2: T = rffi.SHORT
-        elif bytes == -4: T = rffi.INT
-        elif bytes == -8: T = rffi.LONGLONG
-        else: raise NotImplementedError(bytes)
+        T = self._get_int_type_from_size(bytes)
         x = llop.gc_load_indexed(T, struct, index, scale, base_ofs)
         return lltype.cast_primitive(lltype.Signed, x)
 
@@ -735,6 +747,30 @@
         return llop.gc_load_indexed(longlong.FLOATSTORAGE,
                                     struct, index, scale, base_ofs)
 
+    def bh_gc_store_indexed_i(self, struct, index, val, scale, base_ofs, bytes,
+                              descr):
+        T = self._get_int_type_from_size(bytes)
+        val = lltype.cast_primitive(T, val)
+        if descr.A.OF == lltype.SingleFloat:
+            val = longlong.int2singlefloat(val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+    def bh_gc_store_indexed_f(self, struct, index, val, scale, base_ofs, bytes,
+                              descr):
+        if bytes != 8:
+            raise Exception("gc_store_indexed_f is only for 'double'!")
+        val = longlong.getrealfloat(val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+    def bh_gc_store_indexed(self, struct, index, val, scale, base_ofs, bytes,
+                            descr):
+        if descr.A.OF == lltype.Float:
+            self.bh_gc_store_indexed_f(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
+        else:
+            self.bh_gc_store_indexed_i(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
+
     def bh_increment_debug_counter(self, addr):
         p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
         p[0] += 1
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -496,6 +496,7 @@
     @specialize.argtype(1)
     def write_float_at_mem(self, gcref, ofs, newvalue):
         llop.raw_store(lltype.Void, gcref, ofs, newvalue)
+    write_float_at_mem._annenforceargs_ = [None, None, None, longlong.r_float_storage]
 
     # ____________________________________________________________
 
@@ -754,6 +755,16 @@
         offset = base_ofs + scale * index
         return self.read_float_at_mem(addr, offset)
 
+    def bh_gc_store_indexed_i(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_int_at_mem(addr, offset, bytes, val)
+
+    def bh_gc_store_indexed_f(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_float_at_mem(addr, offset, val)
+
     def bh_new(self, sizedescr):
         return self.gc_ll_descr.gc_malloc(sizedescr)
 
diff --git a/rpython/jit/backend/x86/test/test_llop.py b/rpython/jit/backend/x86/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_llop.py
@@ -0,0 +1,14 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(Jit386Mixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+
+    # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+    # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+    # symbolic as the offset. It is not a problem because it is tested anyway
+    # by the same test in test_metainterp.py
+    TEST_BLACKHOLE = False
+
diff --git a/rpython/jit/backend/x86/test/test_strstorage.py b/rpython/jit/backend/x86/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_strstorage.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.metainterp.test.test_strstorage import TestStrStorage as _TestStrStorage
-
-
-class TestStrStorage(Jit386Mixin, _TestStrStorage):
-    # for the individual tests see
-    # ====> ../../../metainterp/test/test_strstorage.py
-    pass
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1131,6 +1131,26 @@
                               [op.args[0], op.args[1],
                                op.args[2], op.args[3], c_bytes], op.result)
 
+    def rewrite_op_gc_store_indexed(self, op):
+        T = op.args[2].concretetype
+        kind = getkind(T)[0]
+        assert kind != 'r'
+        descr = self.cpu.arraydescrof(rffi.CArray(T))
+        if (not isinstance(op.args[3], Constant) or
+            not isinstance(op.args[4], Constant)):
+            raise NotImplementedError("gc_store_indexed: 'scale' and 'base_ofs'"
+                                      " should be constants")
+        # According to the comment in resoperation.py, "itemsize is not signed
+        # (always > 0)", so we don't need the "bytes = -bytes" line which is
+        # in rewrite_op_gc_load_indexed
+        bytes = descr.get_item_size_in_bytes()
+        c_bytes = Constant(bytes, lltype.Signed)
+        return SpaceOperation('gc_store_indexed_%s' % kind,
+                              [op.args[0], op.args[1], op.args[2],
+                               op.args[3], op.args[4], c_bytes, descr], None)
+
+
+
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
         if isinstance(arg0, Constant) and not arg0.value:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1478,6 +1478,18 @@
     def bhimpl_gc_load_indexed_f(cpu, addr, index, scale, base_ofs, bytes):
         return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
 
+    @arguments("cpu", "r", "i", "i", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
+
+    @arguments("cpu", "r", "i", "f", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
+
     @arguments("r", "d", "d")
     def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
         pass
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -251,6 +251,25 @@
     else:
         return BoxInt(cpu.bh_raw_load_i(addr, offset, arraydescr))
 
+def do_gc_store_indexed(cpu, _, addrbox, indexbox, valuebox, scalebox,
+                        base_ofsbox, bytesbox, arraydescr):
+    addr = addrbox.getref_base()
+    index = indexbox.getint()
+    scale = scalebox.getint()
+    base_ofs = base_ofsbox.getint()
+    bytes = bytesbox.getint()
+    if arraydescr.is_array_of_pointers():
+        raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
+    elif arraydescr.is_array_of_floats():
+        floatval = valuebox.getfloatstorage()
+        cpu.bh_gc_store_indexed_f(addr, index, floatval, scale, base_ofs, bytes,
+                                  arraydescr)
+    else:
+        intval = valuebox.getint()
+        cpu.bh_gc_store_indexed_i(addr, index, intval, scale, base_ofs, bytes,
+                                  arraydescr)
+
+
 def exec_new_with_vtable(cpu, descr):
     return cpu.bh_new_with_vtable(descr)
 
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -827,6 +827,21 @@
                             self._remove_symbolics(scalebox),
                             self._remove_symbolics(baseofsbox), bytesbox)
 
+    @arguments("box", "box", "box", "box", "box", "box", "descr")
+    def _opimpl_gc_store_indexed(self, addrbox, indexbox, valuebox,
+                                 scalebox, baseofsbox, bytesbox,
+                                 arraydescr):
+        return self.execute_with_descr(rop.GC_STORE_INDEXED,
+                                       arraydescr,
+                                       addrbox,
+                                       indexbox,
+                                       valuebox,
+                                       self._remove_symbolics(scalebox),
+                                       self._remove_symbolics(baseofsbox),
+                                       bytesbox)
+    opimpl_gc_store_indexed_i = _opimpl_gc_store_indexed
+    opimpl_gc_store_indexed_f = _opimpl_gc_store_indexed
+
     @arguments("box")
     def opimpl_hint_force_virtualizable(self, box):
         self.metainterp.gen_store_back_in_vable(box)
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -19,7 +19,8 @@
                   supports_floats=True,
                   supports_longlong=False,
                   supports_singlefloats=False,
-                  translationoptions={}, **kwds):
+                  translationoptions={},
+                  backendopt_inline_threshold=0, **kwds):
     from rpython.jit.codewriter import support
 
     class FakeJitCell(object):
@@ -59,7 +60,7 @@
         FakeWarmRunnerState.enable_opts = {}
 
     func._jit_unroll_safe_ = True
-    rtyper = support.annotate(func, values,
+    rtyper = support.annotate(func, values, inline=backendopt_inline_threshold,
                               translationoptions=translationoptions)
     graphs = rtyper.annotator.translator.graphs
     testself.all_graphs = graphs
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -0,0 +1,72 @@
+import py
+import sys
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.test_llop import (BaseLLOpTest, str_gc_load,
+                                           newlist_and_gc_store)
+from rpython.jit.codewriter import longlong
+from rpython.jit.metainterp.history import getkind
+from rpython.jit.metainterp.test.support import LLJitMixin
+
+
+class TestLLOp(BaseLLOpTest, LLJitMixin):
+
+    # for the individual tests see
+    # ====> ../../../rtyper/test/test_llop.py
+    TEST_BLACKHOLE = True
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        def f(offset):
+            return str_gc_load(TYPE, buf, offset)
+        res = self.interp_operations(f, [offset], supports_singlefloats=True)
+        #
+        kind = getkind(TYPE)[0] # 'i' or 'f'
+        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
+                                       'finish': 1})
+        #
+        if TYPE == lltype.SingleFloat:
+            # interp_operations returns the int version of r_singlefloat, but
+            # our tests expects to receive an r_singlefloat: let's convert it
+            # back!
+            return longlong.int2singlefloat(res)
+        return res
+
+    def newlist_and_gc_store(self, TYPE, value, expected):
+        def f(value):
+            lst = newlist_and_gc_store(TYPE, value)
+            got = ''.join(lst)
+            if got != expected:
+                # I'm not sure why, but if I use an assert, the test doesn't fail
+                raise ValueError('got != expected')
+            return len(got)
+        #
+        if self.TEST_BLACKHOLE:
+            # we pass a big inline_threshold to ensure that
+            # newlist_and_gc_store is inlined, else the blackhole does not see
+            # (and thus we do not test!) the llop.gc_store_indexed
+            threshold = 33
+        else:
+            threshold = 0
+        return self.interp_operations(f, [value], supports_singlefloats=True,
+                                      backendopt_inline_threshold=threshold)
+
+
+    def test_force_virtual_str_storage(self):
+        byteorder = sys.byteorder
+        size = rffi.sizeof(lltype.Signed)
+        def f(val):
+            if byteorder == 'little':
+                x = chr(val) + '\x00'*(size-1)
+            else:
+                x = '\x00'*(size-1) + chr(val)
+            return str_gc_load(lltype.Signed, x, 0)
+        res = self.interp_operations(f, [42], supports_singlefloats=True)
+        assert res == 42
+        self.check_operations_history({
+            'newstr': 1,              # str forcing
+            'strsetitem': 1,          # str forcing
+            'call_pure_r': 1,         # str forcing (copystrcontent)
+            'guard_no_exception': 1,  # str forcing
+            'gc_load_indexed_i': 1,   # str_storage_getitem
+            'finish': 1
+            })
diff --git a/rpython/jit/metainterp/test/test_strstorage.py b/rpython/jit/metainterp/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/metainterp/test/test_strstorage.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import py
-import sys
-import struct
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.strstorage import str_storage_getitem
-from rpython.rlib.test.test_strstorage import BaseStrStorageTest
-from rpython.jit.codewriter import longlong
-from rpython.jit.metainterp.history import getkind
-from rpython.jit.metainterp.test.support import LLJitMixin
-
-class TestStrStorage(BaseStrStorageTest, LLJitMixin):
-
-    # for the individual tests see
-    # ====> ../../../rlib/test/test_strstorage.py
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        def f():
-            return str_storage_getitem(TYPE, buf, offset)
-        res = self.interp_operations(f, [], supports_singlefloats=True)
-        #
-        kind = getkind(TYPE)[0] # 'i' or 'f'
-        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
-                                       'finish': 1})
-        #
-        if TYPE == lltype.SingleFloat:
-            # interp_operations returns the int version of r_singlefloat, but
-            # our tests expects to receive an r_singlefloat: let's convert it
-            # back!
-            return longlong.int2singlefloat(res)
-        return res
-
-    #def str_storage_supported(self, TYPE):
-    #    py.test.skip('this is not a JIT test')
-
-    def test_force_virtual_str_storage(self):
-        byteorder = sys.byteorder
-        size = rffi.sizeof(lltype.Signed)
-        def f(val):
-            if byteorder == 'little':
-                x = chr(val) + '\x00'*(size-1)
-            else:
-                x = '\x00'*(size-1) + chr(val)
-            return str_storage_getitem(lltype.Signed, x, 0)
-        res = self.interp_operations(f, [42], supports_singlefloats=True)
-        assert res == 42
-        self.check_operations_history({
-            'newstr': 1,              # str forcing
-            'strsetitem': 1,          # str forcing
-            'call_pure_r': 1,         # str forcing (copystrcontent)
-            'guard_no_exception': 1,  # str forcing
-            'gc_load_indexed_i': 1,   # str_storage_getitem
-            'finish': 1
-            })
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -1,13 +1,55 @@
 """
 Buffer protocol support.
 """
-from rpython.rlib.rgc import (
-    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rlist import LIST_OF
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+                              nonmoving_raw_ptr_for_resizable_list,
+                              ll_for_resizable_list)
 from rpython.rlib.signature import signature
 from rpython.rlib import types
+from rpython.rlib import rawstorage
+
+ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
+
+ at specialize.ll()
+def is_alignment_correct(TYPE, index):
+    if ALLOW_UNALIGNED_ACCESS:
+        return True
+    try:
+        rawstorage._check_alignment(TYPE, index)
+    except rawstorage.AlignmentError:
+        return False
+    else:
+        return True
+
+
+class CannotRead(Exception):
+    """
+    Exception raised by Buffer.typed_read in case it is not possible to
+    accomplish the request. This might be because it is not supported by the
+    specific type of buffer, or because of alignment issues.
+    """
+
+class CannotWrite(Exception):
+    """
+    Raised by Buffer.typed_write in case it is not possible to accomplish the
+    request
+    """
 
 class Buffer(object):
-    """Base class for buffers of bytes"""
+    """
+    Base class for buffers of bytes.
+
+    Most probably, you do NOT want to use this as a lone base class, but
+    either inherit from RawBuffer or GCBuffer, so that you automatically get
+    the proper implementation of typed_read and typed_write.
+    """
     _attrs_ = ['readonly']
     _immutable_ = True
 
@@ -25,14 +67,6 @@
         # May be overridden.
         return self.getslice(0, self.getlength(), 1, self.getlength())
 
-    def as_str_and_offset_maybe(self):
-        """
-        If the buffer is backed by a string, return a pair (string, offset),
-        where offset is the offset inside the string where the buffer start.
-        Else, return (None, 0).
-        """
-        return None, 0
-
     def getitem(self, index):
         "Returns the index'th character in the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.
@@ -60,7 +94,141 @@
         for i in range(len(string)):
             self.setitem(start + i, string[i])
 
-class ByteBuffer(Buffer):
+    @jit.look_inside_iff(lambda self, index, count:
+                         jit.isconstant(count) and count <= 8)
+    def setzeros(self, index, count):
+        for i in range(index, index+count):
+            self.setitem(i, '\x00')
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        """
+        Read the value of type TP starting at byte_offset. No bounds checks
+        """
+        raise CannotRead
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        """
+        Write the value of type TP at byte_offset. No bounds checks
+        """
+        raise CannotWrite
+
+
+class RawBuffer(Buffer):
+    """
+    A buffer which is baked by a raw, non-movable memory area. It implementes
+    typed_read and typed_write in terms of get_raw_address(), llop.raw_load,
+    llop.raw_store.
+
+    NOTE: this assumes that get_raw_address() is cheap. Do not use this as a
+    base class if get_raw_address() is potentially costly, like for example if
+    you call rgc.nonmoving_raw_ptr_for_resizable_list
+    """
+    _immutable_ = True
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        """
+        Read the value of type TP starting at byte_offset. No bounds checks
+        """
+        if not is_alignment_correct(TP, byte_offset):
+            raise CannotRead
+        ptr = self.get_raw_address()
+        return llop.raw_load(TP, ptr, byte_offset)
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        """
+        Write the value of type TP at byte_offset. No bounds checks
+        """
+        if self.readonly or not is_alignment_correct(TP, byte_offset):
+            raise CannotWrite
+        ptr = self.get_raw_address()
+        value = lltype.cast_primitive(TP, value)
+        return llop.raw_store(lltype.Void, ptr, byte_offset, value)
+
+
+class GCBuffer(Buffer):
+    """
+    Base class for a buffer which is baked by a GC-managed memory area. You
+    MUST also decorate the class with @GCBuffer.decorate: it implements
+    typed_read and typed_write in terms of llop.gc_load_indexed and
+    llop.gc_store_indexed.
+    """
+    _attrs_ = ['readonly', 'value']
+    _immutable_ = True
+
+    @staticmethod
+    def decorate(targetcls):
+        """
+        Create and attach specialized versions of typed_{read,write}. We need to
+        do this becase the JIT codewriters mandates that base_ofs is an
+        RPython constant.
+        """
+        if targetcls.__bases__ != (GCBuffer,):
+            raise ValueError("@GCBuffer.decorate should be used only on "
+                             "GCBuffer subclasses")
+
+        base_ofs = targetcls._get_gc_data_offset()
+        scale_factor = llmemory.sizeof(lltype.Char)
+
+        @specialize.ll_and_arg(1)
+        def typed_read(self, TP, byte_offset):
+            if not is_alignment_correct(TP, byte_offset):
+                raise CannotRead
+            lldata = self._get_gc_data()
+            byte_offset += self._get_gc_data_extra_offset()
+            return llop.gc_load_indexed(TP, lldata, byte_offset,
+                                        scale_factor, base_ofs)
+
+        @specialize.ll_and_arg(1)
+        def typed_write(self, TP, byte_offset, value):
+            if self.readonly or not is_alignment_correct(TP, byte_offset):
+                raise CannotWrite
+            lldata = self._get_gc_data()
+            byte_offset += self._get_gc_data_extra_offset()
+            value = lltype.cast_primitive(TP, value)
+            return llop.gc_store_indexed(lltype.Void, lldata, byte_offset, value,
+                                         scale_factor, base_ofs)
+
+        targetcls.typed_read = typed_read
+        targetcls.typed_write = typed_write
+        return targetcls
+
+    @staticmethod
+    def _get_gc_data_offset(self):
+        raise NotImplementedError
+
+    def _get_gc_data_extra_offset(self):
+        return 0
+
+    def _get_gc_data(self):
+        raise NotImplementedError
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        raise NotImplementedError("You MUST decorate this class with "
+                                  "@GCBuffer.decorate")
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        raise NotImplementedError("You MUST decorate this class with "
+                                  "@GCBuffer.decorate")
+
+
+def get_gc_data_for_list_of_chars(data):
+    ll_data = ll_for_resizable_list(data)
+    ll_items = ll_data.items
+    return lltype.cast_opaque_ptr(llmemory.GCREF, ll_items)
+
+def get_gc_data_offset_for_list_of_chars():
+    LIST = LIST_OF(lltype.Char)
+    return llmemory.itemoffsetof(LIST.items.TO, 0)
+
+
+ at GCBuffer.decorate
+class ByteBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, n):
@@ -79,11 +247,21 @@
     def get_raw_address(self):
         return nonmoving_raw_ptr_for_resizable_list(self.data)
 
-class StringBuffer(Buffer):
+    def _get_gc_data(self):
+        return get_gc_data_for_list_of_chars(self.data)
+
+    @staticmethod
+    def _get_gc_data_offset():
+        return get_gc_data_offset_for_list_of_chars()
+
+
+ at GCBuffer.decorate
+class StringBuffer(GCBuffer):
     _attrs_ = ['readonly', 'value']
     _immutable_ = True
 
     def __init__(self, value):
+        assert value  is not None
         self.value = value
         self.readonly = 1
 
@@ -93,9 +271,6 @@
     def as_str(self):
         return self.value
 
-    def as_str_and_offset_maybe(self):
-        return self.value, 0
-
     def getitem(self, index):
         return self.value[index]
 
@@ -114,6 +289,16 @@
         # may still raise ValueError on some GCs
         return rffi.get_raw_address_of_string(self.value)
 
+    @staticmethod
+    def _get_gc_data_offset():
+        return (llmemory.offsetof(STR, 'chars') +
+                llmemory.itemoffsetof(STR.chars, 0))
+
+    def _get_gc_data(self):
+        lls = llstr(self.value)
+        return lltype.cast_opaque_ptr(llmemory.GCREF, lls)
+
+
 class SubBuffer(Buffer):
     _attrs_ = ['buffer', 'offset', 'size', 'readonly']
     _immutable_ = True
@@ -147,12 +332,6 @@
         else:
             return 0
 
-    def as_str_and_offset_maybe(self):
-        string, offset = self.buffer.as_str_and_offset_maybe()
-        if string is not None:
-            return string, offset + self.offset
-        return None, 0
-
     def getitem(self, index):
         return self.buffer.getitem(self.offset + index)
 
@@ -176,3 +355,11 @@
         from rpython.rtyper.lltypesystem import rffi
         ptr = self.buffer.get_raw_address()
         return rffi.ptradd(ptr, self.offset)
+
+    @specialize.ll_and_arg(1)
+    def typed_read(self, TP, byte_offset):
+        return self.buffer.typed_read(TP, byte_offset + self.offset)
+
+    @specialize.ll_and_arg(1)
+    def typed_write(self, TP, byte_offset, value):
+        return self.buffer.typed_write(TP, byte_offset + self.offset, value)
diff --git a/rpython/rlib/mutbuffer.py b/rpython/rlib/mutbuffer.py
new file mode 100644
--- /dev/null
+++ b/rpython/rlib/mutbuffer.py
@@ -0,0 +1,57 @@
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR, mallocstr
+from rpython.rtyper.annlowlevel import llstr, hlstr
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.buffer import GCBuffer
+from rpython.rlib import jit
+
+ at GCBuffer.decorate
+class MutableStringBuffer(GCBuffer):
+    """
+    A writeable buffer to incrementally fill a string of a fixed size.
+
+    You can fill the string by calling setitem, setslice and typed_write, and
+    get the result by calling finish().
+
+    After you call finish(), you can no longer modify the buffer. There is no
+    check, you will probably get a segfault after translation.
+
+    You can call finish() only once.
+    """
+    _attrs_ = ['readonly', 'll_val', 'size']
+    _immutable_ = True
+
+    def __init__(self, size):
+        self.readonly = False
+        self.size = size
+        self.ll_val = mallocstr(size)
+
+    def getlength(self):
+        return self.size
+
+    def finish(self):
+        if not self.ll_val:
+            raise ValueError("Cannot call finish() twice")
+        result = hlstr(self.ll_val)
+        self.ll_val = lltype.nullptr(STR)
+        self.readonly = True
+        return result
+
+    def as_str(self):
+        raise ValueError('as_str() is not supported. Use finish() instead')
+
+    def _hlstr(self):
+        assert not we_are_translated() # debug only
+        return hlstr(self.ll_val)
+
+    def setitem(self, index, char):
+        self.ll_val.chars[index] = char
+
+    @staticmethod
+    def _get_gc_data_offset():
+        return (llmemory.offsetof(STR, 'chars') +
+                llmemory.itemoffsetof(STR.chars, 0))
+
+    def _get_gc_data(self):
+        return lltype.cast_opaque_ptr(llmemory.GCREF, self.ll_val)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -1051,17 +1051,17 @@
     rgc.nonmoving_raw_ptr_for_resizable_list() might be
     used if needed.  For now, only supports lists of chars.
     """
-    __slots__ = ('_raw_items',)   # either None or a rffi.CCHARP
+    __slots__ = ('_ll_list',)   # either None or a struct of TYPE=LIST_OF(Char)
 
     def __init__(self, lst):
-        self._raw_items = None
+        self._ll_list = None
         self.__from_list(lst)
 
     def __resize(self):
         """Called before an operation changes the size of the list"""
-        if self._raw_items is not None:
+        if self._ll_list is not None:
             list.__init__(self, self.__as_list())
-            self._raw_items = None
+            self._ll_list = None
 
     def __from_list(self, lst):
         """Initialize the list from a copy of the list 'lst'."""
@@ -1072,46 +1072,46 @@
             return
         if len(self) != len(lst):
             self.__resize()
-        if self._raw_items is None:
+        if self._ll_list is None:
             list.__init__(self, lst)
         else:
-            assert len(self) == self._raw_items._obj.getlength() == len(lst)
+            assert len(self) == self._ll_list.length == len(lst)
             for i in range(len(self)):
-                self._raw_items[i] = lst[i]
+                self._ll_list.items[i] = lst[i]
 
     def __as_list(self):
         """Return a list (the same or a different one) which contains the
         items in the regular way."""
-        if self._raw_items is None:
+        if self._ll_list is None:
             return self
-        length = self._raw_items._obj.getlength()
+        length = self._ll_list.length
         assert length == len(self)
-        return [self._raw_items[i] for i in range(length)]
+        return [self._ll_list.items[i] for i in range(length)]
 
     def __getitem__(self, index):
-        if self._raw_items is None:
+        if self._ll_list is None:
             return list.__getitem__(self, index)
         if index < 0:
             index += len(self)
         if not (0 <= index < len(self)):
             raise IndexError
-        return self._raw_items[index]
+        return self._ll_list.items[index]
 
     def __setitem__(self, index, new):
-        if self._raw_items is None:
+        if self._ll_list is None:
             return list.__setitem__(self, index, new)
         if index < 0:
             index += len(self)
         if not (0 <= index < len(self)):
             raise IndexError
-        self._raw_items[index] = new
+        self._ll_list.items[index] = new
 
     def __delitem__(self, index):
         self.__resize()
         list.__delitem__(self, index)
 
     def __getslice__(self, i, j):
-        return list.__getslice__(self.__as_list(), i, j)
+        return self.__class__(list.__getslice__(self.__as_list(), i, j))
 
     def __setslice__(self, i, j, new):
         lst = self.__as_list()
@@ -1218,15 +1218,23 @@
         list.sort(lst, *args, **kwds)
         self.__from_list(lst)
 
+    def _get_ll_list(self):
+        from rpython.rtyper.lltypesystem import rffi
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
+        if self._ll_list is None:
+            LIST = LIST_OF(lltype.Char)
+            existing_items = list(self)
+            n = len(self)
+            self._ll_list = lltype.malloc(LIST, immortal=True)
+            self._ll_list.length = n
+            self._ll_list.items = lltype.malloc(LIST.items.TO, n)
+            self.__from_list(existing_items)
+            assert self._ll_list is not None
+        return self._ll_list
+
     def _nonmoving_raw_ptr_for_resizable_list(self):
-        if self._raw_items is None:
-            existing_items = list(self)
-            from rpython.rtyper.lltypesystem import lltype, rffi
-            self._raw_items = lltype.malloc(rffi.CCHARP.TO, len(self),
-                                           flavor='raw', immortal=True)
-            self.__from_list(existing_items)
-            assert self._raw_items is not None
-        return self._raw_items
+        ll_list = self._get_ll_list()
+        return ll_nonmovable_raw_ptr_for_resizable_list(ll_list)
 
 def resizable_list_supporting_raw_ptr(lst):
     return _ResizableListSupportingRawPtr(lst)
@@ -1235,6 +1243,18 @@
     assert isinstance(lst, _ResizableListSupportingRawPtr)
     return lst._nonmoving_raw_ptr_for_resizable_list()
 
+def ll_for_resizable_list(lst):
+    """
+    This is the equivalent of llstr(), but for lists. It can be called only if
+    the list has been created by calling resizable_list_supporting_raw_ptr().
+
+    In theory, all the operations on lst are immediately visible also on
+    ll_list. However, support for that is incomplete in
+    _ResizableListSupportingRawPtr and as such, the pointer becomes invalid as
+    soon as you call a resizing operation on lst.
+    """
+    assert isinstance(lst, _ResizableListSupportingRawPtr)
+    return lst._get_ll_list()
 
 def _check_resizable_list_of_chars(s_list):
     from rpython.annotator import model as annmodel
@@ -1256,6 +1276,10 @@
         return s_list
 
     def specialize_call(self, hop):
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
+        if hop.args_r[0].LIST != LIST_OF(lltype.Char):
+            raise ValueError('Resizable list of chars does not have the '
+                             'expected low-level type')
         hop.exception_cannot_occur()
         return hop.inputarg(hop.args_r[0], 0)
 
@@ -1274,6 +1298,24 @@
         return hop.gendirectcall(ll_nonmovable_raw_ptr_for_resizable_list,
                                  v_list)
 
+class Entry(ExtRegistryEntry):
+    _about_ = ll_for_resizable_list
+
+    def compute_result_annotation(self, s_list):
+        from rpython.rtyper.lltypesystem.rlist import LIST_OF
+        from rpython.rtyper.llannotation import lltype_to_annotation
+        _check_resizable_list_of_chars(s_list)
+        LIST = LIST_OF(lltype.Char)
+        return lltype_to_annotation(lltype.Ptr(LIST))
+
+    def specialize_call(self, hop):
+        hop.exception_cannot_occur()
+        assert hop.args_r[0].lowleveltype == hop.r_result.lowleveltype
+        v_ll_list, = hop.inputargs(*hop.args_r)
+        return hop.genop('same_as', [v_ll_list],
+                         resulttype = hop.r_result.lowleveltype)
+
+
 @jit.dont_look_inside
 def ll_nonmovable_raw_ptr_for_resizable_list(ll_list):
     """
diff --git a/rpython/rlib/rstruct/ieee.py b/rpython/rlib/rstruct/ieee.py
--- a/rpython/rlib/rstruct/ieee.py
+++ b/rpython/rlib/rstruct/ieee.py
@@ -264,15 +264,23 @@
     sign = r_ulonglong(sign)
     return (mant, (sign << BITS - MANT_DIG - 1) | exp)
 
+
+def pack_float(wbuf, pos, x, size, be):
+    unsigned = float_pack(x, size)
+    value = rarithmetic.longlongmask(unsigned)
+    pack_float_to_buffer(wbuf, pos, value, size, be)
+
 @jit.unroll_safe
-def pack_float(result, x, size, be):
-    l = []
-    unsigned = float_pack(x, size)
-    for i in range(size):
-        l.append(chr((unsigned >> (i * 8)) & 0xFF))
+def pack_float_to_buffer(wbuf, pos, value, size, be):
     if be:
-        l.reverse()
-    result.append("".join(l))
+        # write in reversed order
+        for i in range(size):
+            c = chr((value >> (i * 8)) & 0xFF)
+            wbuf.setitem(pos + size - i - 1, c)
+    else:
+        for i in range(size):
+            c = chr((value >> (i * 8)) & 0xFF)
+            wbuf.setitem(pos+i, c)
 
 @jit.unroll_safe
 def pack_float80(result, x, size, be):
diff --git a/rpython/rlib/rstruct/nativefmttable.py b/rpython/rlib/rstruct/nativefmttable.py
--- a/rpython/rlib/rstruct/nativefmttable.py
+++ b/rpython/rlib/rstruct/nativefmttable.py
@@ -6,10 +6,11 @@
 
 from rpython.rlib import jit, longlong2float
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rarithmetic import r_singlefloat, widen
+from rpython.rlib.rarithmetic import r_singlefloat, widen, intmask
 from rpython.rlib.rstruct import standardfmttable as std
 from rpython.rlib.rstruct.standardfmttable import native_is_bigendian
 from rpython.rlib.rstruct.error import StructError
+from rpython.rlib.rstruct.ieee import pack_float_to_buffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.tool import rffi_platform
@@ -26,35 +27,26 @@
 # ____________________________________________________________
 
 
-range_8_unroll = unrolling_iterable(list(reversed(range(8))))
-range_4_unroll = unrolling_iterable(list(reversed(range(4))))
-
 def pack_double(fmtiter):
     doubleval = fmtiter.accept_float_arg()
+    if std.pack_fastpath(rffi.DOUBLE)(fmtiter, doubleval):
+        return
+    # slow path
     value = longlong2float.float2longlong(doubleval)
-    if fmtiter.bigendian:
-        for i in range_8_unroll:
-            x = (value >> (8*i)) & 0xff
-            fmtiter.result.append(chr(x))
-    else:
-        for i in range_8_unroll:
-            fmtiter.result.append(chr(value & 0xff))
-            value >>= 8
-
+    pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 8, fmtiter.bigendian)
+    fmtiter.advance(8)
 
 def pack_float(fmtiter):
     doubleval = fmtiter.accept_float_arg()
     floatval = r_singlefloat(doubleval)
+    if std.pack_fastpath(rffi.FLOAT)(fmtiter, floatval):
+        return
+    # slow path
     value = longlong2float.singlefloat2uint(floatval)
     value = widen(value)
-    if fmtiter.bigendian:
-        for i in range_4_unroll:
-            x = (value >> (8*i)) & 0xff
-            fmtiter.result.append(chr(x))
-    else:
-        for i in range_4_unroll:
-            fmtiter.result.append(chr(value & 0xff))
-            value >>= 8
+    value = intmask(value)
+    pack_float_to_buffer(fmtiter.wbuf, fmtiter.pos, value, 4, fmtiter.bigendian)
+    fmtiter.advance(4)
 
 # ____________________________________________________________
 #
@@ -151,7 +143,8 @@
     if len(unistr) != 1:
         raise StructError("expected a unicode string of length 1")
     c = unistr[0]   # string->char conversion for the annotator
-    unichar.pack_unichar(c, fmtiter.result)
+    unichar.pack_unichar(c, fmtiter.wbuf, fmtiter.pos)
+    fmtiter.advance(unichar.UNICODE_SIZE)
 
 @specialize.argtype(0)
 def unpack_unichar(fmtiter):
diff --git a/rpython/rlib/rstruct/runpack.py b/rpython/rlib/rstruct/runpack.py
--- a/rpython/rlib/rstruct/runpack.py
+++ b/rpython/rlib/rstruct/runpack.py
@@ -8,19 +8,28 @@
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib.buffer import StringBuffer
 
 class MasterReader(object):
     def __init__(self, s):
-        self.input = s
+        self.inputbuf = StringBuffer(s)
+        self.length = len(s)
         self.inputpos = 0
 
+    def can_advance(self, count):
+        end = self.inputpos + count
+        return end <= self.length
+
+    def advance(self, count):
+        if not self.can_advance(count):
+            raise StructError("unpack str size too short for format")
+        self.inputpos += count
+
     def read(self, count):
-        end = self.inputpos + count
-        if end > len(self.input):
-            raise StructError("unpack str size too short for format")
-        s = self.input[self.inputpos : end]
-        self.inputpos = end
-        return s
+        curpos = self.inputpos
+        end = curpos + count
+        self.advance(count) # raise if we are out of bound
+        return self.inputbuf.getslice(curpos, end, 1, count)
 
     def align(self, mask):
         self.inputpos = (self.inputpos + mask) & ~mask
@@ -40,11 +49,14 @@
         def appendobj(self, value):
             self.value = value
 
-        def get_buffer_as_string_maybe(self):
-            return self.mr.input, self.mr.inputpos
+        def get_buffer_and_pos(self):
+            return self.mr.inputbuf, self.mr.inputpos
 
-        def skip(self, size):
-            self.read(size) # XXX, could avoid taking the slice
+        def can_advance(self, size):
+            return self.mr.can_advance(size)
+
+        def advance(self, size):
+            self.mr.advance(size)
     ReaderForPos.__name__ = 'ReaderForPos%d' % pos
     return ReaderForPos
 
diff --git a/rpython/rlib/rstruct/standardfmttable.py b/rpython/rlib/rstruct/standardfmttable.py
--- a/rpython/rlib/rstruct/standardfmttable.py
+++ b/rpython/rlib/rstruct/standardfmttable.py
@@ -12,34 +12,84 @@
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.unroll import unrolling_iterable
-from rpython.rlib.strstorage import str_storage_getitem
+from rpython.rlib.buffer import StringBuffer
 from rpython.rlib import rarithmetic
+from rpython.rlib.buffer import CannotRead, CannotWrite
 from rpython.rtyper.lltypesystem import rffi
 
+USE_FASTPATH = True    # set to False by some tests
+ALLOW_SLOWPATH = True  # set to False by some tests
+ALLOW_FASTPATH = True  # set to False by some tests
+
 native_is_bigendian = struct.pack("=i", 1) == struct.pack(">i", 1)
 native_is_ieee754 = float.__getformat__('double').startswith('IEEE')
 
+ at specialize.memo()
+def pack_fastpath(TYPE):
+    """
+    Create a fast path packer for TYPE. The packer returns True is it succeded
+    or False otherwise.
+    """
+    @specialize.argtype(0)
+    def do_pack_fastpath(fmtiter, value):
+        size = rffi.sizeof(TYPE)
+        if (not USE_FASTPATH or
+            fmtiter.bigendian != native_is_bigendian or
+            not native_is_ieee754):
+            raise CannotWrite
+        #
+        # typed_write() might raise CannotWrite
+        fmtiter.wbuf.typed_write(TYPE, fmtiter.pos, value)
+        if not ALLOW_FASTPATH:
+            # if we are here it means that typed_write did not raise, and thus
+            # the fast path was actually taken
+            raise ValueError("fastpath not allowed :(")
+        fmtiter.advance(size)
+    #
+    @specialize.argtype(0)
+    def do_pack_fastpath_maybe(fmtiter, value):
+        try:
+            do_pack_fastpath(fmtiter, value)
+        except CannotWrite:
+            if not ALLOW_SLOWPATH:
+                raise ValueError("fastpath not taken :(")
+            return False
+        else:
+            return True
+    #
+    return do_pack_fastpath_maybe
+
 def pack_pad(fmtiter, count):
-    fmtiter.result.append_multiple_char('\x00', count)
+    fmtiter.wbuf.setzeros(fmtiter.pos, count)
+    fmtiter.advance(count)
 
 def pack_char(fmtiter):
     string = fmtiter.accept_str_arg()
     if len(string) != 1:
         raise StructError("expected a string of length 1")
     c = string[0]   # string->char conversion for the annotator
-    fmtiter.result.append(c)
+    fmtiter.wbuf.setitem(fmtiter.pos, c)
+    fmtiter.advance(1)
 
 def pack_bool(fmtiter):
     c = '\x01' if fmtiter.accept_bool_arg() else '\x00'
-    fmtiter.result.append(c)
+    fmtiter.wbuf.setitem(fmtiter.pos, c)
+    fmtiter.advance(1)
+
+def _pack_string(fmtiter, string, count):
+    pos = fmtiter.pos
+    if len(string) < count:
+        n = len(string)
+        fmtiter.wbuf.setslice(pos, string)
+        fmtiter.wbuf.setzeros(pos+n, count-n)
+    else:
+        assert count >= 0
+        fmtiter.wbuf.setslice(pos, string[:count])
+    fmtiter.advance(count)
 
 def pack_string(fmtiter, count):
     string = fmtiter.accept_str_arg()
-    if len(string) < count:
-        fmtiter.result.append(string)
-        fmtiter.result.append_multiple_char('\x00', count - len(string))
-    else:
-        fmtiter.result.append_slice(string, 0, count)
+    _pack_string(fmtiter, string, count)
 
 def pack_pascal(fmtiter, count):
     string = fmtiter.accept_str_arg()
@@ -49,21 +99,28 @@
         if prefix < 0:
             raise StructError("bad '0p' in struct format")
     if prefix > 255:
-        prefixchar = '\xff'
-    else:
-        prefixchar = chr(prefix)
-    fmtiter.result.append(prefixchar)
-    fmtiter.result.append_slice(string, 0, prefix)
-    fmtiter.result.append_multiple_char('\x00', count - (1 + prefix))
+        prefix = 255
+    fmtiter.wbuf.setitem(fmtiter.pos, chr(prefix))
+    fmtiter.advance(1)
+    _pack_string(fmtiter, string, count-1)
 
-def make_float_packer(size):
+
+def make_float_packer(TYPE):
+    size = rffi.sizeof(TYPE)
     def packer(fmtiter):
         fl = fmtiter.accept_float_arg()
+        if TYPE is not rffi.FLOAT and pack_fastpath(TYPE)(fmtiter, fl):
+            return
+        # slow path
         try:
-            return ieee.pack_float(fmtiter.result, fl, size, fmtiter.bigendian)
+            result = ieee.pack_float(fmtiter.wbuf, fmtiter.pos,
+                                     fl, size, fmtiter.bigendian)
         except OverflowError:
             assert size == 4
             raise StructOverflowError("float too large for format 'f'")
+        else:
+            fmtiter.advance(size)
+            return result
     return packer
 
 # ____________________________________________________________
@@ -111,42 +168,56 @@
     errormsg = "argument out of range for %d-byte%s integer format" % (size,
                                                                        plural)
     unroll_revrange_size = unrolling_iterable(range(size-1, -1, -1))
+    TYPE = get_rffi_int_type(size, signed)
 
     def pack_int(fmtiter):
         method = getattr(fmtiter, accept_method)
         value = method()
         if not min <= value <= max:
             raise StructError(errormsg)
+        #
+        if pack_fastpath(TYPE)(fmtiter, value):
+            return
+        #
+        pos = fmtiter.pos + size - 1        
         if fmtiter.bigendian:
             for i in unroll_revrange_size:
                 x = (value >> (8*i)) & 0xff
-                fmtiter.result.append(chr(x))
+                fmtiter.wbuf.setitem(pos-i, chr(x))
         else:
+

From pypy.commits at gmail.com  Wed May 24 12:07:20 2017
From: pypy.commits at gmail.com (Dodan)
Date: Wed, 24 May 2017 09:07:20 -0700 (PDT)
Subject: [pypy-commit] pypy PyPy_profopt_enabled: PyPy 5.8.0 functional
 profopt. Precleanup
Message-ID: <5925afb8.59b0df0a.252b5.9c56@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: PyPy_profopt_enabled
Changeset: r91398:8fe61093e999
Date: 2017-05-03 13:23 +0300
http://bitbucket.org/pypy/pypy/changeset/8fe61093e999/

Log:	PyPy 5.8.0 functional profopt. Precleanup

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -264,11 +264,11 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if (config.translation.profopt is not None
-                    and not config.translation.noprofopt):
-                raise Exception("Cannot use the --profopt option "
-                                "when --shared is on (it is by default). "
-                                "See issue #2398.")
+            # if (config.translation.profopt is not None
+            #         and not config.translation.noprofopt):
+            #     raise Exception("Cannot use the --profopt option "
+            #                     "when --shared is on (it is by default). "
+            #                     "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -142,7 +142,7 @@
     BoolOption("verbose", "Print extra information", default=False,
                cmdline="--verbose"),
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
-    StrOption("profopt", "Specify profile based optimization script",
+    BoolOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),
     BoolOption("noprofopt", "Don't use profile based optimization",
                default=False, cmdline="--no-profopt", negation=False),
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -87,11 +87,11 @@
     def _do_profbased(self):
         ProfDriver, args = self.profbased
         profdrv = ProfDriver(self)
-        dolog = getattr(log, profdrv.name)
+        # dolog = getattr(log, profdrv.name)
         dolog(args)
         exename = profdrv.first()
-        dolog('Gathering profile data from: %s %s' % (
-            str(exename), args))
+        # dolog('Gathering profile data from: %s %s' % (
+        #     str(exename), args))
         profdrv.probe(exename, args)
         return profdrv.after()
 
@@ -391,6 +391,8 @@
         shared = self.config.translation.shared
 
         extra_opts = []
+        if self.config.translation.profopt and not self.config.translation.noprofopt:
+            extra_opts += ["profopt"]
         if self.config.translation.make_jobs != 1:
             extra_opts += ['-j', str(self.config.translation.make_jobs)]
         if self.config.translation.lldebug:
@@ -420,11 +422,14 @@
             shared=self.config.translation.shared,
             config=self.config)
 
-        if self.has_profopt():
-            profopt = self.config.translation.profopt
-            mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
-            mk.definition('DEFAULT_TARGET', 'profopt')
-            mk.definition('PROFOPT', profopt)
+
+        #if self.has_profopt():
+        #     profopt = self.config.translation.profopt
+        #     mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
+        #     mk.definition('DEFAULT_TARGET', 'profopt')
+        #     mk.definition('PROFOPT', profopt)
+        # I have commented this out because I have made another rule with other definitions explained below
+
 
         rules = [
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
@@ -433,15 +438,30 @@
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
-            ]
-        if self.has_profopt():
-            rules.append(
-                ('profopt', '', [
-                '$(MAKENOPROF)',
-                '$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" $(TARGET)',
-                'cd $(RPYDIR)/translator/goal && $(ABS_TARGET) $(PROFOPT)',
+        ]
+        # if self.has_profopt():
+
+        #added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument
+        # PROFOPT_Tar
+
+        mk.definition('PROFOPT_TARGET','pypy-c')
+        mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o',
+                     '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS) -lgcov')
+        rules.append(
+            ('profopt', '', [
+                '$(MAKE) CFLAGS="-fprofile-generate -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-generate $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+                'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so || true',
+                'cp -f libpypy-c.so $(RPYDIR)/../pypy/goal/ || true',
+                'rm -f $(RPYDIR)/../pypy/goal/pypy-c',
+                'cp -f pypy-c $(RPYDIR)/../pypy/goal/',
+                '$(RPYDIR)/../pypy/goal/pypy-c $(RPYDIR)/../pypy/goal/regrtest/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true',
                 '$(MAKE) clean_noprof',
-                '$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" $(TARGET)']))
+                '$(MAKE) CFLAGS="-fprofile-use -fprofile-correction -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-use $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+                'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so || true',
+                'cp -f libpypy-c.so $(RPYDIR)/../pypy/goal/ || true',
+                'rm -f $(RPYDIR)/../pypy/goal/pypy-c',
+                'cp -f pypy-c $(RPYDIR)/../pypy/goal/']))
+
         for rule in rules:
             mk.rule(*rule)
 

From pypy.commits at gmail.com  Wed May 24 12:07:22 2017
From: pypy.commits at gmail.com (Dodan)
Date: Wed, 24 May 2017 09:07:22 -0700 (PDT)
Subject: [pypy-commit] pypy PyPy_profopt_enabled:
 PyPy-5.8.0-Profopt-enabled-postclean
Message-ID: <5925afba.ce1a1c0a.36e46.c337@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: PyPy_profopt_enabled
Changeset: r91399:f1aa2fc615ef
Date: 2017-05-04 09:38 +0300
http://bitbucket.org/pypy/pypy/changeset/f1aa2fc615ef/

Log:	PyPy-5.8.0-Profopt-enabled-postclean

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -264,11 +264,6 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            # if (config.translation.profopt is not None
-            #         and not config.translation.noprofopt):
-            #     raise Exception("Cannot use the --profopt option "
-            #                     "when --shared is on (it is by default). "
-            #                     "See issue #2398.")
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -144,8 +144,6 @@
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
     BoolOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),
-    BoolOption("noprofopt", "Don't use profile based optimization",
-               default=False, cmdline="--no-profopt", negation=False),
     BoolOption("instrument", "internal: turn instrumentation on",
                default=False, cmdline=None),
     BoolOption("countmallocs", "Count mallocs and frees", default=False,
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -33,29 +33,6 @@
     return python
 
 
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        return self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        self.compiler.platform.execute(exe, args)
-
-    def after(self):
-        return self.build('-fprofile-use')
-
-    def build(self, option):
-        eci = ExternalCompilationInfo(compile_extra=[option],
-                                      link_extra=[option])
-        return self.compiler._build(eci)
-
 class CCompilerDriver(object):
     def __init__(self, platform, cfiles, eci, outputfilename=None,
                  profbased=False):
@@ -65,7 +42,7 @@
         self.cfiles = cfiles
         self.eci = eci
         self.outputfilename = outputfilename
-        self.profbased = profbased
+        # self.profbased = profbased
 
     def _build(self, eci=ExternalCompilationInfo(), shared=False):
         outputfilename = self.outputfilename
@@ -79,22 +56,6 @@
                                      outputfilename=outputfilename,
                                      standalone=not shared)
 
-    def build(self, shared=False):
-        if self.profbased:
-            return self._do_profbased()
-        return self._build(shared=shared)
-
-    def _do_profbased(self):
-        ProfDriver, args = self.profbased
-        profdrv = ProfDriver(self)
-        # dolog = getattr(log, profdrv.name)
-        dolog(args)
-        exename = profdrv.first()
-        # dolog('Gathering profile data from: %s %s' % (
-        #     str(exename), args))
-        profdrv.probe(exename, args)
-        return profdrv.after()
-
 class CBuilder(object):
     c_source_filename = None
     _compiled = False
@@ -259,27 +220,6 @@
     _entrypoint_wrapper = None
     make_entrypoint_wrapper = True    # for tests
 
-    def getprofbased(self):
-        profbased = None
-        if self.config.translation.instrumentctl is not None:
-            profbased = self.config.translation.instrumentctl
-        else:
-            # xxx handling config.translation.profopt is a bit messy, because
-            # it could be an empty string (not to be confused with None) and
-            # because noprofopt can be used as an override.
-            profopt = self.config.translation.profopt
-            if profopt is not None and not self.config.translation.noprofopt:
-                profbased = (ProfOpt, profopt)
-        return profbased
-
-    def has_profopt(self):
-        profbased = self.getprofbased()
-        retval = (profbased and isinstance(profbased, tuple)
-                and profbased[0] is ProfOpt)
-        if retval and self.translator.platform.name == 'msvc':
-            raise ValueError('Cannot do profile based optimization on MSVC,'
-                    'it is not supported in free compiler version')
-        return retval
 
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct
@@ -391,7 +331,7 @@
         shared = self.config.translation.shared
 
         extra_opts = []
-        if self.config.translation.profopt and not self.config.translation.noprofopt:
+        if self.config.translation.profopt:
             extra_opts += ["profopt"]
         if self.config.translation.make_jobs != 1:
             extra_opts += ['-j', str(self.config.translation.make_jobs)]
@@ -423,14 +363,6 @@
             config=self.config)
 
 
-        #if self.has_profopt():
-        #     profopt = self.config.translation.profopt
-        #     mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
-        #     mk.definition('DEFAULT_TARGET', 'profopt')
-        #     mk.definition('PROFOPT', profopt)
-        # I have commented this out because I have made another rule with other definitions explained below
-
-
         rules = [
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
             ('debug_exc', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DDO_LOG_EXC" debug_target'),
@@ -439,28 +371,32 @@
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
         ]
-        # if self.has_profopt():
 
-        #added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument
-        # PROFOPT_Tar
+        # added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument
+        # Also made a difference between translating with shared or not, because this affects profopt's target
 
-        mk.definition('PROFOPT_TARGET','pypy-c')
-        mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o',
+
+        if self.config.translation.shared:
+            mk.definition('PROFOPT_TARGET', 'pypy-c')
+            mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o',
                      '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS) -lgcov')
+        else:
+            mk.definition('PROFOPT_TARGET', '$(TARGET)')
+
         rules.append(
             ('profopt', '', [
                 '$(MAKE) CFLAGS="-fprofile-generate -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-generate $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
-                'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so || true',
+                'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so',
                 'cp -f libpypy-c.so $(RPYDIR)/../pypy/goal/ || true',
                 'rm -f $(RPYDIR)/../pypy/goal/pypy-c',
-                'cp -f pypy-c $(RPYDIR)/../pypy/goal/',
+                'cp -f pypy-c $(RPYDIR)/../pypy/goal/ || true',
                 '$(RPYDIR)/../pypy/goal/pypy-c $(RPYDIR)/../pypy/goal/regrtest/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true',
                 '$(MAKE) clean_noprof',
                 '$(MAKE) CFLAGS="-fprofile-use -fprofile-correction -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-use $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
                 'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so || true',
                 'cp -f libpypy-c.so $(RPYDIR)/../pypy/goal/ || true',
                 'rm -f $(RPYDIR)/../pypy/goal/pypy-c',
-                'cp -f pypy-c $(RPYDIR)/../pypy/goal/']))
+                'cp -f pypy-c $(RPYDIR)/../pypy/goal/ || true']))
 
         for rule in rules:
             mk.rule(*rule)
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -227,7 +227,7 @@
         data = cbuilder.cmdexec('hi there')
         assert map(float, data.split()) == [0.0, 0.0]
 
-    def test_profopt(self):
+    def test_profopt1(self):
         if sys.platform == 'win32':
             py.test.skip("no profopt on win32")
         def add(a,b):
@@ -242,19 +242,22 @@
             return 0
         from rpython.translator.interactive import Translation
         # XXX this is mostly a "does not crash option"
-        t = Translation(entry_point, backend='c', profopt="100")
+        t = Translation(entry_point, backend='c', profopt=True, shared=True)
         # no counters
         t.backendopt()
         exe = t.compile()
-        out = py.process.cmdexec("%s 500" % exe)
-        assert int(out) == 500*501/2
-        t = Translation(entry_point, backend='c', profopt="100",
-                        noprofopt=True)
+        assert (os.path.isfile("%s/pypy-c" % os.path.dirname(str(exe))))
+        # out = py.process.cmdexec("%s 500" % exe)
+        # assert int(out) == 500*501/2
+        t = Translation(entry_point, backend='c', profopt=True, shared=False)
         # no counters
         t.backendopt()
         exe = t.compile()
-        out = py.process.cmdexec("%s 500" % exe)
-        assert int(out) == 500*501/2
+        assert (os.path.isfile("%s" % exe))
+        # assert( ("%s/../pypy-c" % exe.purebasename) == "aa")
+
+        # out = py.process.cmdexec("%s 500" % exe)
+        # assert int(out) == 500*501/2
 
     if hasattr(os, 'setpgrp'):
         def test_os_setpgrp(self):
@@ -279,13 +282,12 @@
             return 0
         from rpython.translator.interactive import Translation
         # XXX this is mostly a "does not crash option"
-        t = Translation(entry_point, backend='c', profopt="")
+        t = Translation(entry_point, backend='c', profopt=True, shared=True)
         # no counters
         t.backendopt()
         exe = t.compile()
         #py.process.cmdexec(exe)
-        t = Translation(entry_point, backend='c', profopt="",
-                        noprofopt=True)
+        t = Translation(entry_point, backend='c', profopt=True, shared=True)
         # no counters
         t.backendopt()
         exe = t.compile()

From pypy.commits at gmail.com  Wed May 24 12:07:28 2017
From: pypy.commits at gmail.com (Dodan)
Date: Wed, 24 May 2017 09:07:28 -0700 (PDT)
Subject: [pypy-commit] pypy PyPy_profopt_enabled: Profopt option now can
 come paired with --profoptpath to specify the training workload
Message-ID: <5925afc0.56691c0a.9218d.65ec@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: PyPy_profopt_enabled
Changeset: r91401:b5c11d10c249
Date: 2017-05-15 15:24 +0300
http://bitbucket.org/pypy/pypy/changeset/b5c11d10c249/

Log:	Profopt option now can come paired with --profoptpath to specify the
	training workload

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -264,6 +264,14 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
+
+        # if both profopt and profoptpath are specified then we keep them as they are with no other changes
+        if config.translation.profopt:
+            if config.translation.profoptpath is None:
+                config.translation.profoptpath = "$(RPYDIR)/../lib-python/2.7/test/regrtest.py"
+        elif config.translation.profoptpath is not None:
+            raise Exception("Cannot use --profoptpath without specifying --profopt as well")
+
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -144,6 +144,7 @@
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
     BoolOption("profopt", "Specify profile based optimization script",
               cmdline="--profopt"),
+    StrOption("profoptpath", "Absolute path to the profile guided optimization training script", cmdline="--profoptpath"),
     BoolOption("instrument", "internal: turn instrumentation on",
                default=False, cmdline=None),
     BoolOption("countmallocs", "Count mallocs and frees", default=False,
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -390,7 +390,7 @@
                 'cp -f libpypy-c.so $(RPYDIR)/../pypy/goal/ || true',
                 'rm -f $(RPYDIR)/../pypy/goal/pypy-c',
                 'cp -f pypy-c $(RPYDIR)/../pypy/goal/ || true',
-                '$(RPYDIR)/../pypy/goal/pypy-c $(RPYDIR)/../pypy/goal/regrtest/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true',
+                '$(RPYDIR)/../pypy/goal/pypy-c %s --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true' % self.config.translation.profoptpath,
                 '$(MAKE) clean_noprof',
                 '$(MAKE) CFLAGS="-fprofile-use -fprofile-correction -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-use $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
                 'rm -f $(RPYDIR)/../pypy/goal/libpypy-c.so || true',
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -227,7 +227,7 @@
         data = cbuilder.cmdexec('hi there')
         assert map(float, data.split()) == [0.0, 0.0]
 
-    def test_profopt1(self):
+    def test_profopt(self):
         if sys.platform == 'win32':
             py.test.skip("no profopt on win32")
         def add(a,b):
@@ -247,6 +247,29 @@
         t.backendopt()
         exe = t.compile()
         assert (os.path.isfile("%s/pypy-c" % os.path.dirname(str(exe))))
+        # test --profoptpath
+        profoptpth = open('dummypythontraining.py', 'w+')
+        profoptpth.close()
+        abspath = os.path.abspath('dummypythontraining.py')
+        t = Translation(entry_point, backend='c', profopt=True, profoptpath=abspath, shared=True)
+        # no counters
+        t.backendopt()
+        exe = t.compile()
+        assert (os.path.isfile("%s/pypy-c" % os.path.dirname(str(exe))))
+
+        os.remove(abspath)
+
+        profoptpth = open('dummypythontraining.py', 'w+')
+        profoptpth.close()
+        abspath = os.path.abspath('dummypythontraining.py')
+        t = Translation(entry_point, backend='c', profopt=True, profoptpath=abspath, shared=False)
+        # no counters
+        t.backendopt()
+        exe = t.compile()
+        assert (os.path.isfile("%s" % exe))
+
+        os.remove(abspath)
+
         # out = py.process.cmdexec("%s 500" % exe)
         # assert int(out) == 500*501/2
         t = Translation(entry_point, backend='c', profopt=True, shared=False)

From pypy.commits at gmail.com  Wed May 24 12:07:26 2017
From: pypy.commits at gmail.com (Dodan)
Date: Wed, 24 May 2017 09:07:26 -0700 (PDT)
Subject: [pypy-commit] pypy PyPy_profopt_enabled:
 PyPy-5.8.0-Profopt-enabled-postclean
Message-ID: <5925afbe.4186df0a.37315.ce70@mx.google.com>

Author: Dodan Mihai <mihai.dodan at gmail.com>
Branch: PyPy_profopt_enabled
Changeset: r91400:ee2fb3e0d9b6
Date: 2017-05-04 09:46 +0300
http://bitbucket.org/pypy/pypy/changeset/ee2fb3e0d9b6/

Log:	PyPy-5.8.0-Profopt-enabled-postclean

diff too long, truncating to 2000 out of 284978 lines

diff --git a/pypy/goal/regrtest/185test.db b/pypy/goal/regrtest/185test.db
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..14cb5e258bc0961fa22527dcb5d947a6a0fd5480
GIT binary patch

[cut]

diff --git a/pypy/goal/regrtest/Sine-1000Hz-300ms.aif b/pypy/goal/regrtest/Sine-1000Hz-300ms.aif
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..bf08f5ce859429eeeab213fd29a201b0d851b9c3
GIT binary patch

[cut]

diff --git a/pypy/goal/regrtest/__init__.py b/pypy/goal/regrtest/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/goal/regrtest/__init__.py
@@ -0,0 +1,1 @@
+# Dummy file to make this directory a package.
diff --git a/pypy/goal/regrtest/_mock_backport.py b/pypy/goal/regrtest/_mock_backport.py
new file mode 100644
--- /dev/null
+++ b/pypy/goal/regrtest/_mock_backport.py
@@ -0,0 +1,2358 @@
+# mock.py
+# Test tools for mocking and patching.
+# Maintained by Michael Foord
+# Backport for other versions of Python available from
+# http://pypi.python.org/pypi/mock
+
+__all__ = (
+    'Mock',
+    'MagicMock',
+    'patch',
+    'sentinel',
+    'DEFAULT',
+    'ANY',
+    'call',
+    'create_autospec',
+    'FILTER_DIR',
+    'NonCallableMock',
+    'NonCallableMagicMock',
+    'mock_open',
+    'PropertyMock',
+)
+
+
+__version__ = '1.0'
+
+
+import __builtin__
+import inspect
+import pprint
+import sys
+
+from types import ModuleType
+from functools import wraps, partial
+
+
+_builtins = {name for name in dir(__builtin__) if not name.startswith('_')}
+
+BaseExceptions = (BaseException,)
+if 'java' in sys.platform:
+    # jython
+    import java
+    BaseExceptions = (BaseException, java.lang.Throwable)
+
+
+FILTER_DIR = True
+
+# Workaround for issue #12370
+# Without this, the __class__ properties wouldn't be set correctly
+_safe_super = super
+
+def _is_instance_mock(obj):
+    # can't use isinstance on Mock objects because they override __class__
+    # The base class for all mocks is NonCallableMock
+    return issubclass(type(obj), NonCallableMock)
+
+
+def _is_exception(obj):
+    return (
+        isinstance(obj, BaseExceptions) or
+        isinstance(obj, type) and issubclass(obj, BaseExceptions)
+    )
+
+
+class _slotted(object):
+    __slots__ = ['a']
+
+
+DescriptorTypes = (
+    type(_slotted.a),
+    property,
+)
+
+
+def _get_signature_object(func, as_instance, eat_self):
+    """
+    Given an arbitrary, possibly callable object, try to create a suitable
+    signature object.
+    Return a (reduced func, signature) tuple, or None.
+    """
+    if isinstance(func, type) and not as_instance:
+        # If it's a type and should be modelled as a type, use __init__.
+        try:
+            func = func.__init__
+        except AttributeError:
+            return None
+        # Skip the `self` argument in __init__
+        eat_self = True
+    elif not isinstance(func, FunctionTypes):
+        # If we really want to model an instance of the passed type,
+        # __call__ should be looked up, not __init__.
+        try:
+            func = func.__call__
+        except AttributeError:
+            return None
+    if eat_self:
+        sig_func = partial(func, None)
+    else:
+        sig_func = func
+    try:
+        return func, inspect.signature(sig_func)
+    except ValueError:
+        # Certain callable types are not supported by inspect.signature()
+        return None
+
+
+def _check_signature(func, mock, skipfirst, instance=False):
+    sig = _get_signature_object(func, instance, skipfirst)
+    if sig is None:
+        return
+    func, sig = sig
+    def checksig(_mock_self, *args, **kwargs):
+        sig.bind(*args, **kwargs)
+    _copy_func_details(func, checksig)
+    type(mock)._mock_check_sig = checksig
+
+
+def _copy_func_details(func, funcopy):
+    funcopy.__name__ = func.__name__
+    funcopy.__doc__ = func.__doc__
+    try:
+        funcopy.__text_signature__ = func.__text_signature__
+    except AttributeError:
+        pass
+    # we explicitly don't copy func.__dict__ into this copy as it would
+    # expose original attributes that should be mocked
+    try:
+        funcopy.__module__ = func.__module__
+    except AttributeError:
+        pass
+    try:
+        funcopy.__defaults__ = func.__defaults__
+    except AttributeError:
+        pass
+    try:
+        funcopy.__kwdefaults__ = func.__kwdefaults__
+    except AttributeError:
+        pass
+
+
+def _callable(obj):
+    if isinstance(obj, type):
+        return True
+    if getattr(obj, '__call__', None) is not None:
+        return True
+    return False
+
+
+def _is_list(obj):
+    # checks for list or tuples
+    # XXXX badly named!
+    return type(obj) in (list, tuple)
+
+
+def _instance_callable(obj):
+    """Given an object, return True if the object is callable.
+    For classes, return True if instances would be callable."""
+    if not isinstance(obj, type):
+        # already an instance
+        return getattr(obj, '__call__', None) is not None
+
+    # *could* be broken by a class overriding __mro__ or __dict__ via
+    # a metaclass
+    for base in (obj,) + obj.__mro__:
+        if base.__dict__.get('__call__') is not None:
+            return True
+    return False
+
+
+def _set_signature(mock, original, instance=False):
+    # creates a function with signature (*args, **kwargs) that delegates to a
+    # mock. It still does signature checking by calling a lambda with the same
+    # signature as the original.
+    if not _callable(original):
+        return
+
+    skipfirst = isinstance(original, type)
+    result = _get_signature_object(original, instance, skipfirst)
+    if result is None:
+        return
+    func, sig = result
+    def checksig(*args, **kwargs):
+        sig.bind(*args, **kwargs)
+    _copy_func_details(func, checksig)
+
+    name = original.__name__
+    if not name.isidentifier():
+        name = 'funcopy'
+    context = {'_checksig_': checksig, 'mock': mock}
+    src = """def %s(*args, **kwargs):
+    _checksig_(*args, **kwargs)
+    return mock(*args, **kwargs)""" % name
+    exec (src, context)
+    funcopy = context[name]
+    _setup_func(funcopy, mock)
+    return funcopy
+
+
+def _setup_func(funcopy, mock):
+    funcopy.mock = mock
+
+    # can't use isinstance with mocks
+    if not _is_instance_mock(mock):
+        return
+
+    def assert_called_with(*args, **kwargs):
+        return mock.assert_called_with(*args, **kwargs)
+    def assert_called_once_with(*args, **kwargs):
+        return mock.assert_called_once_with(*args, **kwargs)
+    def assert_has_calls(*args, **kwargs):
+        return mock.assert_has_calls(*args, **kwargs)
+    def assert_any_call(*args, **kwargs):
+        return mock.assert_any_call(*args, **kwargs)
+    def reset_mock():
+        funcopy.method_calls = _CallList()
+        funcopy.mock_calls = _CallList()
+        mock.reset_mock()
+        ret = funcopy.return_value
+        if _is_instance_mock(ret) and not ret is mock:
+            ret.reset_mock()
+
+    funcopy.called = False
+    funcopy.call_count = 0
+    funcopy.call_args = None
+    funcopy.call_args_list = _CallList()
+    funcopy.method_calls = _CallList()
+    funcopy.mock_calls = _CallList()
+
+    funcopy.return_value = mock.return_value
+    funcopy.side_effect = mock.side_effect
+    funcopy._mock_children = mock._mock_children
+
+    funcopy.assert_called_with = assert_called_with
+    funcopy.assert_called_once_with = assert_called_once_with
+    funcopy.assert_has_calls = assert_has_calls
+    funcopy.assert_any_call = assert_any_call
+    funcopy.reset_mock = reset_mock
+
+    mock._mock_delegate = funcopy
+
+
+def _is_magic(name):
+    return '__%s__' % name[2:-2] == name
+
+
+class _SentinelObject(object):
+    "A unique, named, sentinel object."
+    def __init__(self, name):
+        self.name = name
+
+    def __repr__(self):
+        return 'sentinel.%s' % self.name
+
+
+class _Sentinel(object):
+    """Access attributes to return a named object, usable as a sentinel."""
+    def __init__(self):
+        self._sentinels = {}
+
+    def __getattr__(self, name):
+        if name == '__bases__':
+            # Without this help(unittest.mock) raises an exception
+            raise AttributeError
+        return self._sentinels.setdefault(name, _SentinelObject(name))
+
+
+sentinel = _Sentinel()
+
+DEFAULT = sentinel.DEFAULT
+_missing = sentinel.MISSING
+_deleted = sentinel.DELETED
+
+
+def _copy(value):
+    if type(value) in (dict, list, tuple, set):
+        return type(value)(value)
+    return value
+
+
+_allowed_names = set(
+    [
+        'return_value', '_mock_return_value', 'side_effect',
+        '_mock_side_effect', '_mock_parent', '_mock_new_parent',
+        '_mock_name', '_mock_new_name'
+    ]
+)
+
+
+def _delegating_property(name):
+    _allowed_names.add(name)
+    _the_name = '_mock_' + name
+    def _get(self, name=name, _the_name=_the_name):
+        sig = self._mock_delegate
+        if sig is None:
+            return getattr(self, _the_name)
+        return getattr(sig, name)
+    def _set(self, value, name=name, _the_name=_the_name):
+        sig = self._mock_delegate
+        if sig is None:
+            self.__dict__[_the_name] = value
+        else:
+            setattr(sig, name, value)
+
+    return property(_get, _set)
+
+
+
+class _CallList(list):
+
+    def __contains__(self, value):
+        if not isinstance(value, list):
+            return list.__contains__(self, value)
+        len_value = len(value)
+        len_self = len(self)
+        if len_value > len_self:
+            return False
+
+        for i in range(0, len_self - len_value + 1):
+            sub_list = self[i:i+len_value]
+            if sub_list == value:
+                return True
+        return False
+
+    def __repr__(self):
+        return pprint.pformat(list(self))
+
+
+def _check_and_set_parent(parent, value, name, new_name):
+    if not _is_instance_mock(value):
+        return False
+    if ((value._mock_name or value._mock_new_name) or
+        (value._mock_parent is not None) or
+        (value._mock_new_parent is not None)):
+        return False
+
+    _parent = parent
+    while _parent is not None:
+        # setting a mock (value) as a child or return value of itself
+        # should not modify the mock
+        if _parent is value:
+            return False
+        _parent = _parent._mock_new_parent
+
+    if new_name:
+        value._mock_new_parent = parent
+        value._mock_new_name = new_name
+    if name:
+        value._mock_parent = parent
+        value._mock_name = name
+    return True
+
+# Internal class to identify if we wrapped an iterator object or not.
+class _MockIter(object):
+    def __init__(self, obj):
+        self.obj = iter(obj)
+    def __iter__(self):
+        return self
+    def __next__(self):
+        return next(self.obj)
+
+class Base(object):
+    _mock_return_value = DEFAULT
+    _mock_side_effect = None
+    def __init__(self, *args, **kwargs):
+        pass
+
+
+
+class NonCallableMock(Base):
+    """A non-callable version of `Mock`"""
+
+    def __new__(cls, *args, **kw):
+        # every instance has its own class
+        # so we can create magic methods on the
+        # class without stomping on other mocks
+        new = type(cls.__name__, (cls,), {'__doc__': cls.__doc__})
+        instance = object.__new__(new)
+        return instance
+
+
+    def __init__(
+            self, spec=None, wraps=None, name=None, spec_set=None,
+            parent=None, _spec_state=None, _new_name='', _new_parent=None,
+            _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs
+        ):
+        if _new_parent is None:
+            _new_parent = parent
+
+        __dict__ = self.__dict__
+        __dict__['_mock_parent'] = parent
+        __dict__['_mock_name'] = name
+        __dict__['_mock_new_name'] = _new_name
+        __dict__['_mock_new_parent'] = _new_parent
+
+        if spec_set is not None:
+            spec = spec_set
+            spec_set = True
+        if _eat_self is None:
+            _eat_self = parent is not None
+
+        self._mock_add_spec(spec, spec_set, _spec_as_instance, _eat_self)
+
+        __dict__['_mock_children'] = {}
+        __dict__['_mock_wraps'] = wraps
+        __dict__['_mock_delegate'] = None
+
+        __dict__['_mock_called'] = False
+        __dict__['_mock_call_args'] = None
+        __dict__['_mock_call_count'] = 0
+        __dict__['_mock_call_args_list'] = _CallList()
+        __dict__['_mock_mock_calls'] = _CallList()
+
+        __dict__['method_calls'] = _CallList()
+        __dict__['_mock_unsafe'] = unsafe
+
+        if kwargs:
+            self.configure_mock(**kwargs)
+
+        _safe_super(NonCallableMock, self).__init__(
+            spec, wraps, name, spec_set, parent,
+            _spec_state
+        )
+
+
+    def attach_mock(self, mock, attribute):
+        """
+        Attach a mock as an attribute of this one, replacing its name and
+        parent. Calls to the attached mock will be recorded in the
+        `method_calls` and `mock_calls` attributes of this one."""
+        mock._mock_parent = None
+        mock._mock_new_parent = None
+        mock._mock_name = ''
+        mock._mock_new_name = None
+
+        setattr(self, attribute, mock)
+
+
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+
+
+    def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False,
+                       _eat_self=False):
+        _spec_class = None
+        _spec_signature = None
+
+        if spec is not None and not _is_list(spec):
+            if isinstance(spec, type):
+                _spec_class = spec
+            else:
+                _spec_class = _get_class(spec)
+            res = _get_signature_object(spec,
+                                        _spec_as_instance, _eat_self)
+            _spec_signature = res and res[1]
+
+            spec = dir(spec)
+
+        __dict__ = self.__dict__
+        __dict__['_spec_class'] = _spec_class
+        __dict__['_spec_set'] = spec_set
+        __dict__['_spec_signature'] = _spec_signature
+        __dict__['_mock_methods'] = spec
+
+
+    def __get_return_value(self):
+        ret = self._mock_return_value
+        if self._mock_delegate is not None:
+            ret = self._mock_delegate.return_value
+
+        if ret is DEFAULT:
+            ret = self._get_child_mock(
+                _new_parent=self, _new_name='()'
+            )
+            self.return_value = ret
+        return ret
+
+
+    def __set_return_value(self, value):
+        if self._mock_delegate is not None:
+            self._mock_delegate.return_value = value
+        else:
+            self._mock_return_value = value
+            _check_and_set_parent(self, value, None, '()')
+
+    __return_value_doc = "The value to be returned when the mock is called."
+    return_value = property(__get_return_value, __set_return_value,
+                            __return_value_doc)
+
+
+    @property
+    def __class__(self):
+        if self._spec_class is None:
+            return type(self)
+        return self._spec_class
+
+    called = _delegating_property('called')
+    call_count = _delegating_property('call_count')
+    call_args = _delegating_property('call_args')
+    call_args_list = _delegating_property('call_args_list')
+    mock_calls = _delegating_property('mock_calls')
+
+
+    def __get_side_effect(self):
+        delegated = self._mock_delegate
+        if delegated is None:
+            return self._mock_side_effect
+        sf = delegated.side_effect
+        if sf is not None and not callable(sf) and not isinstance(sf, _MockIter):
+            sf = _MockIter(sf)
+            delegated.side_effect = sf
+        return sf
+
+    def __set_side_effect(self, value):
+        value = _try_iter(value)
+        delegated = self._mock_delegate
+        if delegated is None:
+            self._mock_side_effect = value
+        else:
+            delegated.side_effect = value
+
+    side_effect = property(__get_side_effect, __set_side_effect)
+
+
+    def reset_mock(self):
+        "Restore the mock object to its initial state."
+        self.called = False
+        self.call_args = None
+        self.call_count = 0
+        self.mock_calls = _CallList()
+        self.call_args_list = _CallList()
+        self.method_calls = _CallList()
+
+        for child in self._mock_children.values():
+            if isinstance(child, _SpecState):
+                continue
+            child.reset_mock()
+
+        ret = self._mock_return_value
+        if _is_instance_mock(ret) and ret is not self:
+            ret.reset_mock()
+
+
+    def configure_mock(self, **kwargs):
+        """Set attributes on the mock through keyword arguments.
+
+        Attributes plus return values and side effects can be set on child
+        mocks using standard dot notation and unpacking a dictionary in the
+        method call:
+
+        >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError}
+        >>> mock.configure_mock(**attrs)"""
+        for arg, val in sorted(kwargs.items(),
+                               # we sort on the number of dots so that
+                               # attributes are set before we set attributes on
+                               # attributes
+                               key=lambda entry: entry[0].count('.')):
+            args = arg.split('.')
+            final = args.pop()
+            obj = self
+            for entry in args:
+                obj = getattr(obj, entry)
+            setattr(obj, final, val)
+
+
+    def __getattr__(self, name):
+        if name in {'_mock_methods', '_mock_unsafe'}:
+            raise AttributeError(name)
+        elif self._mock_methods is not None:
+            if name not in self._mock_methods or name in _all_magics:
+                raise AttributeError("Mock object has no attribute %r" % name)
+        elif _is_magic(name):
+            raise AttributeError(name)
+        if not self._mock_unsafe:
+            if name.startswith(('assert', 'assret')):
+                raise AttributeError(name)
+
+        result = self._mock_children.get(name)
+        if result is _deleted:
+            raise AttributeError(name)
+        elif result is None:
+            wraps = None
+            if self._mock_wraps is not None:
+                # XXXX should we get the attribute without triggering code
+                # execution?
+                wraps = getattr(self._mock_wraps, name)
+
+            result = self._get_child_mock(
+                parent=self, name=name, wraps=wraps, _new_name=name,
+                _new_parent=self
+            )
+            self._mock_children[name]  = result
+
+        elif isinstance(result, _SpecState):
+            result = create_autospec(
+                result.spec, result.spec_set, result.instance,
+                result.parent, result.name
+            )
+            self._mock_children[name]  = result
+
+        return result
+
+
+    def __repr__(self):
+        _name_list = [self._mock_new_name]
+        _parent = self._mock_new_parent
+        last = self
+
+        dot = '.'
+        if _name_list == ['()']:
+            dot = ''
+        seen = set()
+        while _parent is not None:
+            last = _parent
+
+            _name_list.append(_parent._mock_new_name + dot)
+            dot = '.'
+            if _parent._mock_new_name == '()':
+                dot = ''
+
+            _parent = _parent._mock_new_parent
+
+            # use ids here so as not to call __hash__ on the mocks
+            if id(_parent) in seen:
+                break
+            seen.add(id(_parent))
+
+        _name_list = list(reversed(_name_list))
+        _first = last._mock_name or 'mock'
+        if len(_name_list) > 1:
+            if _name_list[1] not in ('()', '().'):
+                _first += '.'
+        _name_list[0] = _first
+        name = ''.join(_name_list)
+
+        name_string = ''
+        if name not in ('mock', 'mock.'):
+            name_string = ' name=%r' % name
+
+        spec_string = ''
+        if self._spec_class is not None:
+            spec_string = ' spec=%r'
+            if self._spec_set:
+                spec_string = ' spec_set=%r'
+            spec_string = spec_string % self._spec_class.__name__
+        return "<%s%s%s id='%s'>" % (
+            type(self).__name__,
+            name_string,
+            spec_string,
+            id(self)
+        )
+
+
+    def __dir__(self):
+        """Filter the output of `dir(mock)` to only useful members."""
+        if not FILTER_DIR:
+            return object.__dir__(self)
+
+        extras = self._mock_methods or []
+        from_type = dir(type(self))
+        from_dict = list(self.__dict__)
+
+        from_type = [e for e in from_type if not e.startswith('_')]
+        from_dict = [e for e in from_dict if not e.startswith('_') or
+                     _is_magic(e)]
+        return sorted(set(extras + from_type + from_dict +
+                          list(self._mock_children)))
+
+
+    def __setattr__(self, name, value):
+        if name in _allowed_names:
+            # property setters go through here
+            return object.__setattr__(self, name, value)
+        elif (self._spec_set and self._mock_methods is not None and
+            name not in self._mock_methods and
+            name not in self.__dict__):
+            raise AttributeError("Mock object has no attribute '%s'" % name)
+        elif name in _unsupported_magics:
+            msg = 'Attempting to set unsupported magic method %r.' % name
+            raise AttributeError(msg)
+        elif name in _all_magics:
+            if self._mock_methods is not None and name not in self._mock_methods:
+                raise AttributeError("Mock object has no attribute '%s'" % name)
+
+            if not _is_instance_mock(value):
+                setattr(type(self), name, _get_method(name, value))
+                original = value
+                value = lambda *args, **kw: original(self, *args, **kw)
+            else:
+                # only set _new_name and not name so that mock_calls is tracked
+                # but not method calls
+                _check_and_set_parent(self, value, None, name)
+                setattr(type(self), name, value)
+                self._mock_children[name] = value
+        elif name == '__class__':
+            self._spec_class = value
+            return
+        else:
+            if _check_and_set_parent(self, value, name, name):
+                self._mock_children[name] = value
+        return object.__setattr__(self, name, value)
+
+
+    def __delattr__(self, name):
+        if name in _all_magics and name in type(self).__dict__:
+            delattr(type(self), name)
+            if name not in self.__dict__:
+                # for magic methods that are still MagicProxy objects and
+                # not set on the instance itself
+                return
+
+        if name in self.__dict__:
+            object.__delattr__(self, name)
+
+        obj = self._mock_children.get(name, _missing)
+        if obj is _deleted:
+            raise AttributeError(name)
+        if obj is not _missing:
+            del self._mock_children[name]
+        self._mock_children[name] = _deleted
+
+
+    def _format_mock_call_signature(self, args, kwargs):
+        name = self._mock_name or 'mock'
+        return _format_call_signature(name, args, kwargs)
+
+
+    def _format_mock_failure_message(self, args, kwargs):
+        message = 'Expected call: %s\nActual call: %s'
+        expected_string = self._format_mock_call_signature(args, kwargs)
+        call_args = self.call_args
+        if len(call_args) == 3:
+            call_args = call_args[1:]
+        actual_string = self._format_mock_call_signature(*call_args)
+        return message % (expected_string, actual_string)
+
+
+    def _call_matcher(self, _call):
+        """
+        Given a call (or simply a (args, kwargs) tuple), return a
+        comparison key suitable for matching with other calls.
+        This is a best effort method which relies on the spec's signature,
+        if available, or falls back on the arguments themselves.
+        """
+        sig = self._spec_signature
+        if sig is not None:
+            if len(_call) == 2:
+                name = ''
+                args, kwargs = _call
+            else:
+                name, args, kwargs = _call
+            try:
+                return name, sig.bind(*args, **kwargs)
+            except TypeError as e:
+                return e.with_traceback(None)
+        else:
+            return _call
+
+    def assert_not_called(_mock_self):
+        """assert that the mock was never called.
+        """
+        self = _mock_self
+        if self.call_count != 0:
+            msg = ("Expected '%s' to not have been called. Called %s times." %
+                   (self._mock_name or 'mock', self.call_count))
+            raise AssertionError(msg)
+
+    def assert_called_with(_mock_self, *args, **kwargs):
+        """assert that the mock was called with the specified arguments.
+
+        Raises an AssertionError if the args and keyword args passed in are
+        different to the last call to the mock."""
+        self = _mock_self
+        if self.call_args is None:
+            expected = self._format_mock_call_signature(args, kwargs)
+            raise AssertionError('Expected call: %s\nNot called' % (expected,))
+
+        def _error_message():
+            msg = self._format_mock_failure_message(args, kwargs)
+            return msg
+        expected = self._call_matcher((args, kwargs))
+        actual = self._call_matcher(self.call_args)
+        if expected != actual:
+            raise AssertionError(_error_message())
+
+
+    def assert_called_once_with(_mock_self, *args, **kwargs):
+        """assert that the mock was called exactly once and with the specified
+        arguments."""
+        self = _mock_self
+        if not self.call_count == 1:
+            msg = ("Expected '%s' to be called once. Called %s times." %
+                   (self._mock_name or 'mock', self.call_count))
+            raise AssertionError(msg)
+        return self.assert_called_with(*args, **kwargs)
+
+
+    def assert_has_calls(self, calls, any_order=False):
+        """assert the mock has been called with the specified calls.
+        The `mock_calls` list is checked for the calls.
+
+        If `any_order` is False (the default) then the calls must be
+        sequential. There can be extra calls before or after the
+        specified calls.
+
+        If `any_order` is True then the calls can be in any order, but
+        they must all appear in `mock_calls`."""
+        expected = [self._call_matcher(c) for c in calls]
+        all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls)
+        if not any_order:
+            if expected not in all_calls:
+                raise AssertionError(
+                    'Calls not found.\nExpected: %r\n'
+                    'Actual: %r' % (calls, self.mock_calls)
+                )
+            return
+
+        all_calls = list(all_calls)
+
+        not_found = []
+        for kall in expected:
+            try:
+                all_calls.remove(kall)
+            except ValueError:
+                not_found.append(kall)
+        if not_found:
+            raise AssertionError(
+                '%r not all found in call list' % (tuple(not_found),)
+            )
+
+
+    def assert_any_call(self, *args, **kwargs):
+        """assert the mock has been called with the specified arguments.
+
+        The assert passes if the mock has *ever* been called, unlike
+        `assert_called_with` and `assert_called_once_with` that only pass if
+        the call is the most recent one."""
+        expected = self._call_matcher((args, kwargs))
+        actual = [self._call_matcher(c) for c in self.call_args_list]
+        if expected not in actual:
+            expected_string = self._format_mock_call_signature(args, kwargs)
+            raise AssertionError(
+                '%s call not found' % expected_string
+            )
+
+
+    def _get_child_mock(self, **kw):
+        """Create the child mocks for attributes and return value.
+        By default child mocks will be the same type as the parent.
+        Subclasses of Mock may want to override this to customize the way
+        child mocks are made.
+
+        For non-callable mocks the callable variant will be used (rather than
+        any custom subclass)."""
+        _type = type(self)
+        if not issubclass(_type, CallableMixin):
+            if issubclass(_type, NonCallableMagicMock):
+                klass = MagicMock
+            elif issubclass(_type, NonCallableMock) :
+                klass = Mock
+        else:
+            klass = _type.__mro__[1]
+        return klass(**kw)
+
+
+
+def _try_iter(obj):
+    if obj is None:
+        return obj
+    if _is_exception(obj):
+        return obj
+    if _callable(obj):
+        return obj
+    try:
+        return iter(obj)
+    except TypeError:
+        # XXXX backwards compatibility
+        # but this will blow up on first call - so maybe we should fail early?
+        return obj
+
+
+
+class CallableMixin(Base):
+
+    def __init__(self, spec=None, side_effect=None, return_value=DEFAULT,
+                 wraps=None, name=None, spec_set=None, parent=None,
+                 _spec_state=None, _new_name='', _new_parent=None, **kwargs):
+        self.__dict__['_mock_return_value'] = return_value
+
+        _safe_super(CallableMixin, self).__init__(
+            spec, wraps, name, spec_set, parent,
+            _spec_state, _new_name, _new_parent, **kwargs
+        )
+
+        self.side_effect = side_effect
+
+
+    def _mock_check_sig(self, *args, **kwargs):
+        # stub method that can be replaced with one with a specific signature
+        pass
+
+
+    def __call__(_mock_self, *args, **kwargs):
+        # can't use self in-case a function / method we are mocking uses self
+        # in the signature
+        _mock_self._mock_check_sig(*args, **kwargs)
+        return _mock_self._mock_call(*args, **kwargs)
+
+
+    def _mock_call(_mock_self, *args, **kwargs):
+        self = _mock_self
+        self.called = True
+        self.call_count += 1
+        _new_name = self._mock_new_name
+        _new_parent = self._mock_new_parent
+
+        _call = _Call((args, kwargs), two=True)
+        self.call_args = _call
+        self.call_args_list.append(_call)
+        self.mock_calls.append(_Call(('', args, kwargs)))
+
+        seen = set()
+        skip_next_dot = _new_name == '()'
+        do_method_calls = self._mock_parent is not None
+        name = self._mock_name
+        while _new_parent is not None:
+            this_mock_call = _Call((_new_name, args, kwargs))
+            if _new_parent._mock_new_name:
+                dot = '.'
+                if skip_next_dot:
+                    dot = ''
+
+                skip_next_dot = False
+                if _new_parent._mock_new_name == '()':
+                    skip_next_dot = True
+
+                _new_name = _new_parent._mock_new_name + dot + _new_name
+
+            if do_method_calls:
+                if _new_name == name:
+                    this_method_call = this_mock_call
+                else:
+                    this_method_call = _Call((name, args, kwargs))
+                _new_parent.method_calls.append(this_method_call)
+
+                do_method_calls = _new_parent._mock_parent is not None
+                if do_method_calls:
+                    name = _new_parent._mock_name + '.' + name
+
+            _new_parent.mock_calls.append(this_mock_call)
+            _new_parent = _new_parent._mock_new_parent
+
+            # use ids here so as not to call __hash__ on the mocks
+            _new_parent_id = id(_new_parent)
+            if _new_parent_id in seen:
+                break
+            seen.add(_new_parent_id)
+
+        ret_val = DEFAULT
+        effect = self.side_effect
+        if effect is not None:
+            if _is_exception(effect):
+                raise effect
+
+            if not _callable(effect):
+                result = next(effect)
+                if _is_exception(result):
+                    raise result
+                if result is DEFAULT:
+                    result = self.return_value
+                return result
+
+            ret_val = effect(*args, **kwargs)
+
+        if (self._mock_wraps is not None and
+             self._mock_return_value is DEFAULT):
+            return self._mock_wraps(*args, **kwargs)
+        if ret_val is DEFAULT:
+            ret_val = self.return_value
+        return ret_val
+
+
+
+class Mock(CallableMixin, NonCallableMock):
+    """
+    Create a new `Mock` object. `Mock` takes several optional arguments
+    that specify the behaviour of the Mock object:
+
+    * `spec`: This can be either a list of strings or an existing object (a
+      class or instance) that acts as the specification for the mock object. If
+      you pass in an object then a list of strings is formed by calling dir on
+      the object (excluding unsupported magic attributes and methods). Accessing
+      any attribute not in this list will raise an `AttributeError`.
+
+      If `spec` is an object (rather than a list of strings) then
+      `mock.__class__` returns the class of the spec object. This allows mocks
+      to pass `isinstance` tests.
+
+    * `spec_set`: A stricter variant of `spec`. If used, attempting to *set*
+      or get an attribute on the mock that isn't on the object passed as
+      `spec_set` will raise an `AttributeError`.
+
+    * `side_effect`: A function to be called whenever the Mock is called. See
+      the `side_effect` attribute. Useful for raising exceptions or
+      dynamically changing return values. The function is called with the same
+      arguments as the mock, and unless it returns `DEFAULT`, the return
+      value of this function is used as the return value.
+
+      If `side_effect` is an iterable then each call to the mock will return
+      the next value from the iterable. If any of the members of the iterable
+      are exceptions they will be raised instead of returned.
+
+    * `return_value`: The value returned when the mock is called. By default
+      this is a new Mock (created on first access). See the
+      `return_value` attribute.
+
+    * `wraps`: Item for the mock object to wrap. If `wraps` is not None then
+      calling the Mock will pass the call through to the wrapped object
+      (returning the real result). Attribute access on the mock will return a
+      Mock object that wraps the corresponding attribute of the wrapped object
+      (so attempting to access an attribute that doesn't exist will raise an
+      `AttributeError`).
+
+      If the mock has an explicit `return_value` set then calls are not passed
+      to the wrapped object and the `return_value` is returned instead.
+
+    * `name`: If the mock has a name then it will be used in the repr of the
+      mock. This can be useful for debugging. The name is propagated to child
+      mocks.
+
+    Mocks can also be called with arbitrary keyword arguments. These will be
+    used to set attributes on the mock after it is created.
+    """
+
+
+
+def _dot_lookup(thing, comp, import_path):
+    try:
+        return getattr(thing, comp)
+    except AttributeError:
+        __import__(import_path)
+        return getattr(thing, comp)
+
+
+def _importer(target):
+    components = target.split('.')
+    import_path = components.pop(0)
+    thing = __import__(import_path)
+
+    for comp in components:
+        import_path += ".%s" % comp
+        thing = _dot_lookup(thing, comp, import_path)
+    return thing
+
+
+def _is_started(patcher):
+    # XXXX horrible
+    return hasattr(patcher, 'is_local')
+
+
+class _patch(object):
+
+    attribute_name = None
+    _active_patches = []
+
+    def __init__(
+            self, getter, attribute, new, spec, create,
+            spec_set, autospec, new_callable, kwargs
+        ):
+        if new_callable is not None:
+            if new is not DEFAULT:
+                raise ValueError(
+                    "Cannot use 'new' and 'new_callable' together"
+                )
+            if autospec is not None:
+                raise ValueError(
+                    "Cannot use 'autospec' and 'new_callable' together"
+                )
+
+        self.getter = getter
+        self.attribute = attribute
+        self.new = new
+        self.new_callable = new_callable
+        self.spec = spec
+        self.create = create
+        self.has_local = False
+        self.spec_set = spec_set
+        self.autospec = autospec
+        self.kwargs = kwargs
+        self.additional_patchers = []
+
+
+    def copy(self):
+        patcher = _patch(
+            self.getter, self.attribute, self.new, self.spec,
+            self.create, self.spec_set,
+            self.autospec, self.new_callable, self.kwargs
+        )
+        patcher.attribute_name = self.attribute_name
+        patcher.additional_patchers = [
+            p.copy() for p in self.additional_patchers
+        ]
+        return patcher
+
+
+    def __call__(self, func):
+        if isinstance(func, type):
+            return self.decorate_class(func)
+        return self.decorate_callable(func)
+
+
+    def decorate_class(self, klass):
+        for attr in dir(klass):
+            if not attr.startswith(patch.TEST_PREFIX):
+                continue
+
+            attr_value = getattr(klass, attr)
+            if not hasattr(attr_value, "__call__"):
+                continue
+
+            patcher = self.copy()
+            setattr(klass, attr, patcher(attr_value))
+        return klass
+
+
+    def decorate_callable(self, func):
+        if hasattr(func, 'patchings'):
+            func.patchings.append(self)
+            return func
+
+        @wraps(func)
+        def patched(*args, **keywargs):
+            extra_args = []
+            entered_patchers = []
+
+            exc_info = tuple()
+            try:
+                for patching in patched.patchings:
+                    arg = patching.__enter__()
+                    entered_patchers.append(patching)
+                    if patching.attribute_name is not None:
+                        keywargs.update(arg)
+                    elif patching.new is DEFAULT:
+                        extra_args.append(arg)
+
+                args += tuple(extra_args)
+                return func(*args, **keywargs)
+            except:
+                if (patching not in entered_patchers and
+                    _is_started(patching)):
+                    # the patcher may have been started, but an exception
+                    # raised whilst entering one of its additional_patchers
+                    entered_patchers.append(patching)
+                # Pass the exception to __exit__
+                exc_info = sys.exc_info()
+                # re-raise the exception
+                raise
+            finally:
+                for patching in reversed(entered_patchers):
+                    patching.__exit__(*exc_info)
+
+        patched.patchings = [self]
+        return patched
+
+
+    def get_original(self):
+        target = self.getter()
+        name = self.attribute
+
+        original = DEFAULT
+        local = False
+
+        try:
+            original = target.__dict__[name]
+        except (AttributeError, KeyError):
+            original = getattr(target, name, DEFAULT)
+        else:
+            local = True
+
+        if name in _builtins and isinstance(target, ModuleType):
+            self.create = True
+
+        if not self.create and original is DEFAULT:
+            raise AttributeError(
+                "%s does not have the attribute %r" % (target, name)
+            )
+        return original, local
+
+
+    def __enter__(self):
+        """Perform the patch."""
+        new, spec, spec_set = self.new, self.spec, self.spec_set
+        autospec, kwargs = self.autospec, self.kwargs
+        new_callable = self.new_callable
+        self.target = self.getter()
+
+        # normalise False to None
+        if spec is False:
+            spec = None
+        if spec_set is False:
+            spec_set = None
+        if autospec is False:
+            autospec = None
+
+        if spec is not None and autospec is not None:
+            raise TypeError("Can't specify spec and autospec")
+        if ((spec is not None or autospec is not None) and
+            spec_set not in (True, None)):
+            raise TypeError("Can't provide explicit spec_set *and* spec or autospec")
+
+        original, local = self.get_original()
+
+        if new is DEFAULT and autospec is None:
+            inherit = False
+            if spec is True:
+                # set spec to the object we are replacing
+                spec = original
+                if spec_set is True:
+                    spec_set = original
+                    spec = None
+            elif spec is not None:
+                if spec_set is True:
+                    spec_set = spec
+                    spec = None
+            elif spec_set is True:
+                spec_set = original
+
+            if spec is not None or spec_set is not None:
+                if original is DEFAULT:
+                    raise TypeError("Can't use 'spec' with create=True")
+                if isinstance(original, type):
+                    # If we're patching out a class and there is a spec
+                    inherit = True
+
+            Klass = MagicMock
+            _kwargs = {}
+            if new_callable is not None:
+                Klass = new_callable
+            elif spec is not None or spec_set is not None:
+                this_spec = spec
+                if spec_set is not None:
+                    this_spec = spec_set
+                if _is_list(this_spec):
+                    not_callable = '__call__' not in this_spec
+                else:
+                    not_callable = not callable(this_spec)
+                if not_callable:
+                    Klass = NonCallableMagicMock
+
+            if spec is not None:
+                _kwargs['spec'] = spec
+            if spec_set is not None:
+                _kwargs['spec_set'] = spec_set
+
+            # add a name to mocks
+            if (isinstance(Klass, type) and
+                issubclass(Klass, NonCallableMock) and self.attribute):
+                _kwargs['name'] = self.attribute
+
+            _kwargs.update(kwargs)
+            new = Klass(**_kwargs)
+
+            if inherit and _is_instance_mock(new):
+                # we can only tell if the instance should be callable if the
+                # spec is not a list
+                this_spec = spec
+                if spec_set is not None:
+                    this_spec = spec_set
+                if (not _is_list(this_spec) and not
+                    _instance_callable(this_spec)):
+                    Klass = NonCallableMagicMock
+
+                _kwargs.pop('name')
+                new.return_value = Klass(_new_parent=new, _new_name='()',
+                                         **_kwargs)
+        elif autospec is not None:
+            # spec is ignored, new *must* be default, spec_set is treated
+            # as a boolean. Should we check spec is not None and that spec_set
+            # is a bool?
+            if new is not DEFAULT:
+                raise TypeError(
+                    "autospec creates the mock for you. Can't specify "
+                    "autospec and new."
+                )
+            if original is DEFAULT:
+                raise TypeError("Can't use 'autospec' with create=True")
+            spec_set = bool(spec_set)
+            if autospec is True:
+                autospec = original
+
+            new = create_autospec(autospec, spec_set=spec_set,
+                                  _name=self.attribute, **kwargs)
+        elif kwargs:
+            # can't set keyword args when we aren't creating the mock
+            # XXXX If new is a Mock we could call new.configure_mock(**kwargs)
+            raise TypeError("Can't pass kwargs to a mock we aren't creating")
+
+        new_attr = new
+
+        self.temp_original = original
+        self.is_local = local
+        setattr(self.target, self.attribute, new_attr)
+        if self.attribute_name is not None:
+            extra_args = {}
+            if self.new is DEFAULT:
+                extra_args[self.attribute_name] =  new
+            for patching in self.additional_patchers:
+                arg = patching.__enter__()
+                if patching.new is DEFAULT:
+                    extra_args.update(arg)
+            return extra_args
+
+        return new
+
+
+    def __exit__(self, *exc_info):
+        """Undo the patch."""
+        if not _is_started(self):
+            raise RuntimeError('stop called on unstarted patcher')
+
+        if self.is_local and self.temp_original is not DEFAULT:
+            setattr(self.target, self.attribute, self.temp_original)
+        else:
+            delattr(self.target, self.attribute)
+            if not self.create and not hasattr(self.target, self.attribute):
+                # needed for proxy objects like django settings
+                setattr(self.target, self.attribute, self.temp_original)
+
+        del self.temp_original
+        del self.is_local
+        del self.target
+        for patcher in reversed(self.additional_patchers):
+            if _is_started(patcher):
+                patcher.__exit__(*exc_info)
+
+
+    def start(self):
+        """Activate a patch, returning any created mock."""
+        result = self.__enter__()
+        self._active_patches.append(self)
+        return result
+
+
+    def stop(self):
+        """Stop an active patch."""
+        try:
+            self._active_patches.remove(self)
+        except ValueError:
+            # If the patch hasn't been started this will fail
+            pass
+
+        return self.__exit__()
+
+
+
+def _get_target(target):
+    try:
+        target, attribute = target.rsplit('.', 1)
+    except (TypeError, ValueError):
+        raise TypeError("Need a valid target to patch. You supplied: %r" %
+                        (target,))
+    getter = lambda: _importer(target)
+    return getter, attribute
+
+
+def _patch_object(
+        target, attribute, new=DEFAULT, spec=None,
+        create=False, spec_set=None, autospec=None,
+        new_callable=None, **kwargs
+    ):
+    """
+    patch the named member (`attribute`) on an object (`target`) with a mock
+    object.
+
+    `patch.object` can be used as a decorator, class decorator or a context
+    manager. Arguments `new`, `spec`, `create`, `spec_set`,
+    `autospec` and `new_callable` have the same meaning as for `patch`. Like
+    `patch`, `patch.object` takes arbitrary keyword arguments for configuring
+    the mock object it creates.
+
+    When used as a class decorator `patch.object` honours `patch.TEST_PREFIX`
+    for choosing which methods to wrap.
+    """
+    getter = lambda: target
+    return _patch(
+        getter, attribute, new, spec, create,
+        spec_set, autospec, new_callable, kwargs
+    )
+
+
+def _patch_multiple(target, spec=None, create=False, spec_set=None,
+                    autospec=None, new_callable=None, **kwargs):
+    """Perform multiple patches in a single call. It takes the object to be
+    patched (either as an object or a string to fetch the object by importing)
+    and keyword arguments for the patches::
+
+        with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'):
+            ...
+
+    Use `DEFAULT` as the value if you want `patch.multiple` to create
+    mocks for you. In this case the created mocks are passed into a decorated
+    function by keyword, and a dictionary is returned when `patch.multiple` is
+    used as a context manager.
+
+    `patch.multiple` can be used as a decorator, class decorator or a context
+    manager. The arguments `spec`, `spec_set`, `create`,
+    `autospec` and `new_callable` have the same meaning as for `patch`. These
+    arguments will be applied to *all* patches done by `patch.multiple`.
+
+    When used as a class decorator `patch.multiple` honours `patch.TEST_PREFIX`
+    for choosing which methods to wrap.
+    """
+    if type(target) is str:
+        getter = lambda: _importer(target)
+    else:
+        getter = lambda: target
+
+    if not kwargs:
+        raise ValueError(
+            'Must supply at least one keyword argument with patch.multiple'
+        )
+    # need to wrap in a list for python 3, where items is a view
+    items = list(kwargs.items())
+    attribute, new = items[0]
+    patcher = _patch(
+        getter, attribute, new, spec, create, spec_set,
+        autospec, new_callable, {}
+    )
+    patcher.attribute_name = attribute
+    for attribute, new in items[1:]:
+        this_patcher = _patch(
+            getter, attribute, new, spec, create, spec_set,
+            autospec, new_callable, {}
+        )
+        this_patcher.attribute_name = attribute
+        patcher.additional_patchers.append(this_patcher)
+    return patcher
+
+
+def patch(
+        target, new=DEFAULT, spec=None, create=False,
+        spec_set=None, autospec=None, new_callable=None, **kwargs
+    ):
+    """
+    `patch` acts as a function decorator, class decorator or a context
+    manager. Inside the body of the function or with statement, the `target`
+    is patched with a `new` object. When the function/with statement exits
+    the patch is undone.
+
+    If `new` is omitted, then the target is replaced with a
+    `MagicMock`. If `patch` is used as a decorator and `new` is
+    omitted, the created mock is passed in as an extra argument to the
+    decorated function. If `patch` is used as a context manager the created
+    mock is returned by the context manager.
+
+    `target` should be a string in the form `'package.module.ClassName'`. The
+    `target` is imported and the specified object replaced with the `new`
+    object, so the `target` must be importable from the environment you are
+    calling `patch` from. The target is imported when the decorated function
+    is executed, not at decoration time.
+
+    The `spec` and `spec_set` keyword arguments are passed to the `MagicMock`
+    if patch is creating one for you.
+
+    In addition you can pass `spec=True` or `spec_set=True`, which causes
+    patch to pass in the object being mocked as the spec/spec_set object.
+
+    `new_callable` allows you to specify a different class, or callable object,
+    that will be called to create the `new` object. By default `MagicMock` is
+    used.
+
+    A more powerful form of `spec` is `autospec`. If you set `autospec=True`
+    then the mock with be created with a spec from the object being replaced.
+    All attributes of the mock will also have the spec of the corresponding
+    attribute of the object being replaced. Methods and functions being
+    mocked will have their arguments checked and will raise a `TypeError` if
+    they are called with the wrong signature. For mocks replacing a class,
+    their return value (the 'instance') will have the same spec as the class.
+
+    Instead of `autospec=True` you can pass `autospec=some_object` to use an
+    arbitrary object as the spec instead of the one being replaced.
+
+    By default `patch` will fail to replace attributes that don't exist. If
+    you pass in `create=True`, and the attribute doesn't exist, patch will
+    create the attribute for you when the patched function is called, and
+    delete it again afterwards. This is useful for writing tests against
+    attributes that your production code creates at runtime. It is off by
+    default because it can be dangerous. With it switched on you can write
+    passing tests against APIs that don't actually exist!
+
+    Patch can be used as a `TestCase` class decorator. It works by
+    decorating each test method in the class. This reduces the boilerplate
+    code when your test methods share a common patchings set. `patch` finds
+    tests by looking for method names that start with `patch.TEST_PREFIX`.
+    By default this is `test`, which matches the way `unittest` finds tests.
+    You can specify an alternative prefix by setting `patch.TEST_PREFIX`.
+
+    Patch can be used as a context manager, with the with statement. Here the
+    patching applies to the indented block after the with statement. If you
+    use "as" then the patched object will be bound to the name after the
+    "as"; very useful if `patch` is creating a mock object for you.
+
+    `patch` takes arbitrary keyword arguments. These will be passed to
+    the `Mock` (or `new_callable`) on construction.
+
+    `patch.dict(...)`, `patch.multiple(...)` and `patch.object(...)` are
+    available for alternate use-cases.
+    """
+    getter, attribute = _get_target(target)
+    return _patch(
+        getter, attribute, new, spec, create,
+        spec_set, autospec, new_callable, kwargs
+    )
+
+
+class _patch_dict(object):
+    """
+    Patch a dictionary, or dictionary like object, and restore the dictionary
+    to its original state after the test.
+
+    `in_dict` can be a dictionary or a mapping like container. If it is a
+    mapping then it must at least support getting, setting and deleting items
+    plus iterating over keys.
+
+    `in_dict` can also be a string specifying the name of the dictionary, which
+    will then be fetched by importing it.
+
+    `values` can be a dictionary of values to set in the dictionary. `values`
+    can also be an iterable of `(key, value)` pairs.
+
+    If `clear` is True then the dictionary will be cleared before the new
+    values are set.
+
+    `patch.dict` can also be called with arbitrary keyword arguments to set
+    values in the dictionary::
+
+        with patch.dict('sys.modules', mymodule=Mock(), other_module=Mock()):
+            ...
+
+    `patch.dict` can be used as a context manager, decorator or class
+    decorator. When used as a class decorator `patch.dict` honours
+    `patch.TEST_PREFIX` for choosing which methods to wrap.
+    """
+
+    def __init__(self, in_dict, values=(), clear=False, **kwargs):
+        if isinstance(in_dict, str):
+            in_dict = _importer(in_dict)
+        self.in_dict = in_dict
+        # support any argument supported by dict(...) constructor
+        self.values = dict(values)
+        self.values.update(kwargs)
+        self.clear = clear
+        self._original = None
+
+
+    def __call__(self, f):
+        if isinstance(f, type):
+            return self.decorate_class(f)
+        @wraps(f)
+        def _inner(*args, **kw):
+            self._patch_dict()
+            try:
+                return f(*args, **kw)
+            finally:
+                self._unpatch_dict()
+
+        return _inner
+
+
+    def decorate_class(self, klass):
+        for attr in dir(klass):
+            attr_value = getattr(klass, attr)
+            if (attr.startswith(patch.TEST_PREFIX) and
+                 hasattr(attr_value, "__call__")):
+                decorator = _patch_dict(self.in_dict, self.values, self.clear)
+                decorated = decorator(attr_value)
+                setattr(klass, attr, decorated)
+        return klass
+
+
+    def __enter__(self):
+        """Patch the dict."""
+        self._patch_dict()
+
+
+    def _patch_dict(self):
+        values = self.values
+        in_dict = self.in_dict
+        clear = self.clear
+
+        try:
+            original = in_dict.copy()
+        except AttributeError:
+            # dict like object with no copy method
+            # must support iteration over keys
+            original = {}
+            for key in in_dict:
+                original[key] = in_dict[key]
+        self._original = original
+
+        if clear:
+            _clear_dict(in_dict)
+
+        try:
+            in_dict.update(values)
+        except AttributeError:
+            # dict like object with no update method
+            for key in values:
+                in_dict[key] = values[key]
+
+
+    def _unpatch_dict(self):
+        in_dict = self.in_dict
+        original = self._original
+
+        _clear_dict(in_dict)
+
+        try:
+            in_dict.update(original)
+        except AttributeError:
+            for key in original:
+                in_dict[key] = original[key]
+
+
+    def __exit__(self, *args):
+        """Unpatch the dict."""
+        self._unpatch_dict()
+        return False
+
+    start = __enter__
+    stop = __exit__
+
+
+def _clear_dict(in_dict):
+    try:
+        in_dict.clear()
+    except AttributeError:
+        keys = list(in_dict)
+        for key in keys:
+            del in_dict[key]
+
+
+def _patch_stopall():
+    """Stop all active patches. LIFO to unroll nested patches."""
+    for patch in reversed(_patch._active_patches):
+        patch.stop()
+
+
+patch.object = _patch_object
+patch.dict = _patch_dict
+patch.multiple = _patch_multiple
+patch.stopall = _patch_stopall
+patch.TEST_PREFIX = 'test'
+
+magic_methods = (
+    "lt le gt ge eq ne "
+    "getitem setitem delitem "
+    "len contains iter "
+    "hash str sizeof "
+    "enter exit "
+    "divmod neg pos abs invert "
+    "complex int float index "
+    "trunc floor ceil "
+    "bool next "
+)
+
+numerics = (
+    "add sub mul div floordiv mod lshift rshift and xor or pow truediv"
+)
+inplace = ' '.join('i%s' % n for n in numerics.split())
+right = ' '.join('r%s' % n for n in numerics.split())
+
+# not including __prepare__, __instancecheck__, __subclasscheck__
+# (as they are metaclass methods)
+# __del__ is not supported at all as it causes problems if it exists
+
+_non_defaults = set('__%s__' % method for method in [
+    'get', 'set', 'delete', 'reversed', 'missing', 'reduce', 'reduce_ex',
+    'getinitargs', 'getnewargs', 'getstate', 'setstate', 'getformat',
+    'setformat', 'repr', 'dir', 'subclasses', 'format',
+])
+
+
+def _get_method(name, func):
+    "Turns a callable object (like a mock) into a real function"
+    def method(self, *args, **kw):
+        return func(self, *args, **kw)
+    method.__name__ = name
+    return method
+
+
+_magics = set(
+    '__%s__' % method for method in
+    ' '.join([magic_methods, numerics, inplace, right]).split()
+)
+
+_all_magics = _magics | _non_defaults
+
+_unsupported_magics = set([
+    '__getattr__', '__setattr__',
+    '__init__', '__new__', '__prepare__'
+    '__instancecheck__', '__subclasscheck__',
+    '__del__'
+])
+
+_calculate_return_value = {
+    '__hash__': lambda self: object.__hash__(self),
+    '__str__': lambda self: object.__str__(self),
+    '__sizeof__': lambda self: object.__sizeof__(self),
+}
+
+_return_values = {
+    '__lt__': NotImplemented,
+    '__gt__': NotImplemented,
+    '__le__': NotImplemented,
+    '__ge__': NotImplemented,
+    '__int__': 1,
+    '__contains__': False,
+    '__len__': 0,
+    '__exit__': False,
+    '__complex__': 1j,
+    '__float__': 1.0,
+    '__bool__': True,
+    '__index__': 1,
+}
+
+
+def _get_eq(self):
+    def __eq__(other):
+        ret_val = self.__eq__._mock_return_value
+        if ret_val is not DEFAULT:
+            return ret_val
+        return self is other
+    return __eq__
+
+def _get_ne(self):
+    def __ne__(other):
+        if self.__ne__._mock_return_value is not DEFAULT:
+            return DEFAULT
+        return self is not other
+    return __ne__
+
+def _get_iter(self):
+    def __iter__():
+        ret_val = self.__iter__._mock_return_value
+        if ret_val is DEFAULT:
+            return iter([])
+        # if ret_val was already an iterator, then calling iter on it should
+        # return the iterator unchanged
+        return iter(ret_val)
+    return __iter__
+
+_side_effect_methods = {
+    '__eq__': _get_eq,
+    '__ne__': _get_ne,
+    '__iter__': _get_iter,
+}
+
+
+
+def _set_return_value(mock, method, name):
+    fixed = _return_values.get(name, DEFAULT)
+    if fixed is not DEFAULT:
+        method.return_value = fixed
+        return
+
+    return_calulator = _calculate_return_value.get(name)
+    if return_calulator is not None:
+        try:
+            return_value = return_calulator(mock)
+        except AttributeError:
+            # XXXX why do we return AttributeError here?
+            #      set it as a side_effect instead?
+            return_value = AttributeError(name)
+        method.return_value = return_value
+        return
+
+    side_effector = _side_effect_methods.get(name)
+    if side_effector is not None:
+        method.side_effect = side_effector(mock)
+
+
+
+class MagicMixin(object):
+    def __init__(self, *args, **kw):
+        _safe_super(MagicMixin, self).__init__(*args, **kw)
+        self._mock_set_magics()
+
+
+    def _mock_set_magics(self):
+        these_magics = _magics
+
+        if self._mock_methods is not None:
+            these_magics = _magics.intersection(self._mock_methods)
+
+            remove_magics = set()
+            remove_magics = _magics - these_magics
+
+            for entry in remove_magics:
+                if entry in type(self).__dict__:
+                    # remove unneeded magic methods
+                    delattr(self, entry)
+
+        # don't overwrite existing attributes if called a second time
+        these_magics = these_magics - set(type(self).__dict__)
+
+        _type = type(self)
+        for entry in these_magics:
+            setattr(_type, entry, MagicProxy(entry, self))
+
+
+
+class NonCallableMagicMock(MagicMixin, NonCallableMock):
+    """A version of `MagicMock` that isn't callable."""
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+        self._mock_set_magics()
+
+
+
+class MagicMock(MagicMixin, Mock):
+    """
+    MagicMock is a subclass of Mock with default implementations
+    of most of the magic methods. You can use MagicMock without having to
+    configure the magic methods yourself.
+
+    If you use the `spec` or `spec_set` arguments then *only* magic
+    methods that exist in the spec will be created.
+
+    Attributes and the return value of a `MagicMock` will also be `MagicMocks`.
+    """
+    def mock_add_spec(self, spec, spec_set=False):
+        """Add a spec to a mock. `spec` can either be an object or a
+        list of strings. Only attributes on the `spec` can be fetched as
+        attributes from the mock.
+
+        If `spec_set` is True then only attributes on the spec can be set."""
+        self._mock_add_spec(spec, spec_set)
+        self._mock_set_magics()
+
+
+
+class MagicProxy(object):
+    def __init__(self, name, parent):
+        self.name = name
+        self.parent = parent
+
+    def __call__(self, *args, **kwargs):
+        m = self.create_mock()
+        return m(*args, **kwargs)
+
+    def create_mock(self):
+        entry = self.name
+        parent = self.parent
+        m = parent._get_child_mock(name=entry, _new_name=entry,
+                                   _new_parent=parent)
+        setattr(parent, entry, m)
+        _set_return_value(parent, m, entry)
+        return m
+
+    def __get__(self, obj, _type=None):
+        return self.create_mock()
+
+
+
+class _ANY(object):
+    "A helper object that compares equal to everything."
+
+    __hash__ = object.__hash__
+
+    def __eq__(self, other):
+        return True
+
+    def __ne__(self, other):
+        return False
+
+    def __repr__(self):
+        return '<ANY>'
+
+ANY = _ANY()
+
+
+
+def _format_call_signature(name, args, kwargs):
+    message = '%s(%%s)' % name
+    formatted_args = ''
+    args_string = ', '.join([repr(arg) for arg in args])
+    kwargs_string = ', '.join([
+        '%s=%r' % (key, value) for key, value in sorted(kwargs.items())
+    ])
+    if args_string:
+        formatted_args = args_string
+    if kwargs_string:
+        if formatted_args:
+            formatted_args += ', '
+        formatted_args += kwargs_string
+
+    return message % formatted_args
+
+
+
+class _Call(tuple):
+    """
+    A tuple for holding the results of a call to a mock, either in the form
+    `(args, kwargs)` or `(name, args, kwargs)`.
+
+    If args or kwargs are empty then a call tuple will compare equal to
+    a tuple without those values. This makes comparisons less verbose::
+
+        _Call(('name', (), {})) == ('name',)
+        _Call(('name', (1,), {})) == ('name', (1,))
+        _Call(((), {'a': 'b'})) == ({'a': 'b'},)
+
+    The `_Call` object provides a useful shortcut for comparing with call::
+
+        _Call(((1, 2), {'a': 3})) == call(1, 2, a=3)
+        _Call(('foo', (1, 2), {'a': 3})) == call.foo(1, 2, a=3)
+
+    If the _Call has no name then it will match any name.
+    """
+
+    __hash__ = object.__hash__
+
+    def __new__(cls, value=(), name=None, parent=None, two=False,
+                from_kall=True):
+        name = ''
+        args = ()
+        kwargs = {}
+        _len = len(value)
+        if _len == 3:
+            name, args, kwargs = value
+        elif _len == 2:
+            first, second = value
+            if isinstance(first, str):
+                name = first
+                if isinstance(second, tuple):
+                    args = second
+                else:
+                    kwargs = second
+            else:
+                args, kwargs = first, second
+        elif _len == 1:
+            value, = value
+            if isinstance(value, str):
+                name = value
+            elif isinstance(value, tuple):
+                args = value
+            else:
+                kwargs = value
+
+        if two:
+            return tuple.__new__(cls, (args, kwargs))
+
+        return tuple.__new__(cls, (name, args, kwargs))
+
+
+    def __init__(self, value=(), name=None, parent=None, two=False,
+                 from_kall=True):
+        self.name = name
+        self.parent = parent
+        self.from_kall = from_kall
+
+
+    def __eq__(self, other):
+        if other is ANY:
+            return True
+        try:
+            len_other = len(other)
+        except TypeError:

From pypy.commits at gmail.com  Wed May 24 12:35:44 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 24 May 2017 09:35:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5925b660.e9addf0a.f2209.6e4a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91408:66108bb2353d
Date: 2017-05-24 17:35 +0100
http://bitbucket.org/pypy/pypy/changeset/66108bb2353d/

Log:	hg merge default

diff too long, truncating to 2000 out of 4286 lines

diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -102,6 +102,8 @@
     assert not not_documented
     if branch == 'py3k':
         assert not not_merged
+    else:
+        assert branch in documented, 'Please document this branch before merging: %s' % branch
 
 def test_startrev_on_default():
     doc = ROOT.join('pypy', 'doc')
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -56,3 +56,15 @@
 
 Remove faulty fastpath from ctypes
 
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -23,7 +23,7 @@
 Installing Visual Compiler v9 (for Python 2.7)
 ----------------------------------------------
 
-This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
 was checked in Nov 2016). Note that the compiler suite will be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
--- a/pypy/interpreter/buffer.py
+++ b/pypy/interpreter/buffer.py
@@ -1,5 +1,6 @@
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.buffer import StringBuffer, SubBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 
 from pypy.interpreter.error import oefmt
 
@@ -72,14 +73,15 @@
     def bytes_from_value(self, space, w_val):
         from pypy.module.struct.formatiterator import PackFormatIterator
         itemsize = self.getitemsize()
-        fmtiter = PackFormatIterator(space, [w_val], itemsize)
+        buf = MutableStringBuffer(itemsize)
+        fmtiter = PackFormatIterator(space, buf, [w_val])
         try:
             fmtiter.interpret(self.getformat())
         except StructError as e:
             raise oefmt(space.w_TypeError,
                         "memoryview: invalid type for format '%s'",
                         self.getformat())
-        return fmtiter.result.build()
+        return buf.finish()
 
     def _copy_buffer(self):
         if self.getndim() == 0:
diff --git a/pypy/module/_cffi_backend/cbuffer.py b/pypy/module/_cffi_backend/cbuffer.py
--- a/pypy/module/_cffi_backend/cbuffer.py
+++ b/pypy/module/_cffi_backend/cbuffer.py
@@ -6,13 +6,13 @@
 from pypy.module._cffi_backend import ctypestruct
 from pypy.interpreter.buffer import SimpleView
 
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 
-class LLBuffer(Buffer):
+class LLBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, raw_cdata, size):
@@ -35,7 +35,7 @@
     def getslice(self, start, stop, step, size):
         if step == 1:
             return rffi.charpsize2str(rffi.ptradd(self.raw_cdata, start), size)
-        return Buffer.getslice(self, start, stop, step, size)
+        return RawBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         raw_cdata = rffi.ptradd(self.raw_cdata, start)
diff --git a/pypy/module/_rawffi/buffer.py b/pypy/module/_rawffi/buffer.py
--- a/pypy/module/_rawffi/buffer.py
+++ b/pypy/module/_rawffi/buffer.py
@@ -1,11 +1,10 @@
 from rpython.rtyper.lltypesystem import rffi
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 
 # XXX not the most efficient implementation
 
 
-class RawFFIBuffer(Buffer):
+class RawFFIBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, datainstance):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -367,20 +367,19 @@
         except SocketError as e:
             raise converted_error(space, e)
 
-    @unwrap_spec(level=int, optname=int)
-    def getsockopt_w(self, space, level, optname, w_buflen=None):
+    @unwrap_spec(level=int, optname=int, buflen=int)
+    def getsockopt_w(self, space, level, optname, buflen=0):
         """getsockopt(level, option[, buffersize]) -> value
 
         Get a socket option.  See the Unix manual for level and option.
         If a nonzero buffersize argument is given, the return value is a
         string of that length; otherwise it is an integer.
         """
-        if w_buflen is None:
+        if buflen == 0:
             try:
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
-        buflen = space.int_w(w_buflen)
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -514,6 +514,19 @@
         (reuse,) = struct.unpack('i', reusestr)
         assert reuse != 0
 
+    def test_getsetsockopt_zero(self):
+        # related to issue #2561: when specifying the buffer size param:
+        # if 0 or None, should return the setted value,
+        # otherwise an empty buffer of the specified size
+        import _socket
+        s = _socket.socket()
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+        s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
+        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,11 +1,11 @@
 from rpython.rlib import jit, rgc
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
-from rpython.rlib.buffer import Buffer
 
 from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.baseobjspace import W_Root
@@ -848,7 +848,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayData(Buffer):
+class ArrayData(RawBuffer):
     _immutable_ = True
     readonly = False
     def __init__(self, w_array):
diff --git a/pypy/module/bz2/interp_bz2.py b/pypy/module/bz2/interp_bz2.py
--- a/pypy/module/bz2/interp_bz2.py
+++ b/pypy/module/bz2/interp_bz2.py
@@ -324,6 +324,7 @@
         to compress, call the flush() method to finish the compression process,
         and return what is left in the internal buffers."""
 
+        assert data is not None
         try:
             self.lock()
             datasize = len(data)
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -22,8 +22,7 @@
 from pypy.interpreter.buffer import BufferView
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.argument import Arguments
-
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib.objectmodel import specialize, not_rpython
 from rpython.tool.sourcetools import func_renamer
@@ -428,7 +427,7 @@
 fq = FQ()
 
 
-class CBuffer(Buffer):
+class CBuffer(RawBuffer):
     _immutable_ = True
     def __init__(self, view):
         self.view = view
diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py
--- a/pypy/module/micronumpy/concrete.py
+++ b/pypy/module/micronumpy/concrete.py
@@ -3,7 +3,7 @@
 from rpython.rlib import jit, rgc
 from rpython.rlib.rarithmetic import ovfcheck
 from rpython.rlib.listsort import make_timsort_class
-from rpython.rlib.buffer import Buffer
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.debug import make_sure_not_resized
 from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \
@@ -702,7 +702,8 @@
     def __del__(self):
         free_raw_storage(self.storage)
 
-class ArrayData(Buffer):
+
+class ArrayData(RawBuffer):
     _immutable_ = True
     def __init__(self, impl, readonly):
         self.impl = impl
diff --git a/pypy/module/mmap/interp_mmap.py b/pypy/module/mmap/interp_mmap.py
--- a/pypy/module/mmap/interp_mmap.py
+++ b/pypy/module/mmap/interp_mmap.py
@@ -4,8 +4,8 @@
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.interpreter.buffer import SimpleView
 
-from rpython.rlib.buffer import Buffer
 from rpython.rlib import rmmap, rarithmetic, objectmodel
+from rpython.rlib.buffer import RawBuffer
 from rpython.rlib.rmmap import RValueError, RTypeError, RMMapError
 from rpython.rlib.rstring import StringBuilder
 
@@ -309,7 +309,7 @@
         return OperationError(space.w_SystemError, space.newtext('%s' % e))
 
 
-class MMapBuffer(Buffer):
+class MMapBuffer(RawBuffer):
     _immutable_ = True
 
     def __init__(self, space, mmap, readonly):
@@ -329,7 +329,7 @@
         if step == 1:
             return self.mmap.getslice(start, size)
         else:
-            return Buffer.getslice(self, start, stop, step, size)
+            return RawBuffer.getslice(self, start, stop, step, size)
 
     def setitem(self, index, char):
         self.check_valid_writeable()
diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -34,29 +34,13 @@
             i = 0
             while i < n:
                 i += 1
-                struct.unpack('<i', a)  # ID: unpack
+                struct.unpack('i', a)  # ID: unpack
             return i
         log = self.run(main, [1000])
         assert log.result == 1000
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('unpack', """
             guard_not_invalidated(descr=...)
-            p90 = newstr(4)
-            call_n(ConstClass(copy_raw_to_string), i55, p90, 0, 4, descr=<Callv 0 irii EF=5>)
-            guard_no_exception(descr=...)
-            i91 = strgetitem(p90, 0)
-            i92 = strgetitem(p90, 1)
-            i93 = int_lshift(i92, 8)
-            i94 = int_or(i91, i93)
-            i95 = strgetitem(p90, 2)
-            i96 = int_lshift(i95, 16)
-            i97 = int_or(i94, i96)
-            i98 = strgetitem(p90, 3)
-            i99 = int_ge(i98, 128)
-            guard_false(i99, descr=...)
-            i100 = int_lshift(i98, 24)
-            i101 = int_or(i97, i100)
-            i102 = getfield_raw_i(#, descr=<FieldS pypysig_long_struct.c_value 0>)
-            i103 = int_lt(i102, 0)
-            guard_false(i103, descr=...)
+            i66 = raw_load_i(i53, 0, descr=<ArrayS 4>)
+            --TICK--
         """)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_ffi.py b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
--- a/pypy/module/pypyjit/test_pypy_c/test_ffi.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_ffi.py
@@ -110,38 +110,6 @@
         loops = log.loops_by_id('sleep')
         assert len(loops) == 1 # make sure that we actually JITted the loop
 
-    def test_ctypes_call(self):
-        from rpython.rlib.test.test_clibffi import get_libm_name
-        def main(libm_name):
-            import ctypes
-            libm = ctypes.CDLL(libm_name)
-            fabs = libm.fabs
-            fabs.argtypes = [ctypes.c_double]
-            fabs.restype = ctypes.c_double
-            x = -4
-            i = 0
-            while i < 300:
-                x = fabs(x)
-                x = x - 100
-                i += 1
-            return fabs._ptr.getaddr(), x
-
-        libm_name = get_libm_name(sys.platform)
-        log = self.run(main, [libm_name], import_site=True)
-        fabs_addr, res = log.result
-        assert res == -4.0
-        loop, = log.loops_by_filename(self.filepath)
-        ops = loop.allops()
-        opnames = log.opnames(ops)
-        assert opnames.count('new_with_vtable') == 1 # only the virtualref
-        py.test.skip("XXX re-optimize _ffi for the JIT?")
-        assert opnames.count('call_release_gil') == 1
-        idx = opnames.index('call_release_gil')
-        call = ops[idx]
-        assert (call.args[0] == 'ConstClass(fabs)' or    # e.g. OS/X
-                int(call.args[0]) == fabs_addr)
-
-
     def test__ffi_struct(self):
         def main():
             from _rawffi.alt import _StructDescr, Field, types
diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -30,32 +30,33 @@
         loop, = log.loops_by_filename(self.filepath)
         # This could, of course stand some improvement, to remove all these
         # arithmatic ops, but we've removed all the core overhead.
-        assert loop.match_by_id("pack", """
-            guard_not_invalidated(descr=...)
-            # struct.pack
-            %s
-            i11 = int_and(i4, 255)
-            i13 = int_rshift(i4, 8)
-            i14 = int_and(i13, 255)
-            i16 = int_rshift(i13, 8)
-            i17 = int_and(i16, 255)
-            i19 = int_rshift(i16, 8)
-            i20 = int_and(i19, 255)
-        """ % extra)
+        if sys.byteorder == 'little':
+            # on little endian machines, we take the fast path and store the
+            # value using gc_store_indexed
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                p75 = newstr(4)
+                gc_store_indexed(p75, 0, _, 1, _, 4, descr=...)
+            """ % extra)
+        else:
+            assert loop.match_by_id("pack", """
+                guard_not_invalidated(descr=...)
+                # struct.pack
+                %s
+                i11 = int_and(i4, 255)
+                i13 = int_rshift(i4, 8)
+                i14 = int_and(i13, 255)
+                i16 = int_rshift(i13, 8)
+                i17 = int_and(i16, 255)
+                i19 = int_rshift(i16, 8)
+                i20 = int_and(i19, 255)
+            """ % extra)
 
         if sys.byteorder == 'little':
-            # the newstr and the strsetitems are because the string is forced,
-            # which is in turn because the optimizer doesn't know how to handle a
-            # gc_load_indexed_i on a virtual string. It could be improved, but it
-            # is also true that in real life cases struct.unpack is called on
-            # strings which come from the outside, so it's a minor issue.
             assert loop.match_by_id("unpack", """
                 # struct.unpack
-                p88 = newstr(4)
-                strsetitem(p88, 0, i11)
-                strsetitem(p88, 1, i14)
-                strsetitem(p88, 2, i17)
-                strsetitem(p88, 3, i20)
                 i91 = gc_load_indexed_i(p88, 0, 1, _, -4)
             """)
         else:
@@ -93,27 +94,106 @@
             assert loop.match_by_id('pack', """
                 guard_not_invalidated(descr=...)
                 # struct.pack
+                p85 = newstr(8)
+                gc_store_indexed(p85, 0, -1, 1, _, 4, descr=...)
                 %s
-                i11 = int_and(i4, 255)
-                i13 = int_rshift(i4, 8)
-                i14 = int_and(i13, 255)
-                i16 = int_rshift(i13, 8)
-                i17 = int_and(i16, 255)
-                i19 = int_rshift(i16, 8)
-                i20 = int_and(i19, 255)
+                gc_store_indexed(p85, 4, _, 1, _, 4, descr=...)
             """ % extra)
 
             assert loop.match_by_id('unpack', """
                 # struct.unpack
-                p88 = newstr(8)
-                strsetitem(p88, 0, 255)
-                strsetitem(p88, 1, 255)
-                strsetitem(p88, 2, 255)
-                strsetitem(p88, 3, 255)
-                strsetitem(p88, 4, i11)
-                strsetitem(p88, 5, i14)
-                strsetitem(p88, 6, i17)
-                strsetitem(p88, 7, i20)
                 i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
                 i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
             """)
+
+    def test_unpack_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = array.array('b', buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i65 = raw_load_i(i49, 0, descr=<ArrayS 2>)
+        """)
+
+    def test_unpack_bytearray(self):
+        def main(n):
+            import struct
+            buf = struct.pack('H', 0x1234)
+            buf = bytearray(buf)
+            i = 1
+            res = 0
+            while i < n:
+                val = struct.unpack("h", buf)[0]     # ID: unpack
+                res += val
+                i += 1
+            return res
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        # the offset of gc_load_indexed_i used to be the constant 0. However,
+        # now it is 'i46' because we need to add 0 to
+        # W_BytearrayObject._offset
+        assert loop.match_by_id('unpack', """
+            guard_not_invalidated(descr=...)
+            i70 = gc_load_indexed_i(p48, i46, 1, _, -2)
+        """)
+
+    def test_pack_into_raw_buffer(self):
+        def main(n):
+            import array
+            import struct
+            buf = array.array('b', '\x00'*8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            i65 = int_le(i58, 32767)
+            guard_true(i65, descr=...)
+            raw_store(i55, 4, i58, descr=<ArrayS 2>)
+        """)
+
+    def test_pack_into_bytearray(self):
+        def main(n):
+            import struct
+            buf = bytearray(8)
+            i = 1
+            while i < n:
+                struct.pack_into("h", buf, 4, i)     # ID: pack_into
+                i += 1
+            return i
+        log = self.run(main, [1000])
+        assert log.result == main(1000)
+        loop, = log.loops_by_filename(self.filepath)
+        assert loop.match_by_id('pack_into', """\
+            guard_not_invalidated(descr=...)
+            p68 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__data \d+>)
+            i69 = getfield_gc_i(p68, descr=<FieldS list.length \d+>)
+            i70 = getfield_gc_i(p14, descr=<FieldS pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__offset \d+>)
+            i71 = int_sub(i69, i70)
+            i73 = int_sub(i71, 4)
+            i75 = int_lt(i73, 2)
+            guard_false(i75, descr=...)
+            i77 = int_le(i62, 32767)
+            guard_true(i77, descr=...)
+            p78 = getfield_gc_r(p68, descr=<FieldP list.items \d+>)
+            i81 = int_add(4, i70)
+            gc_store_indexed(p78, i81, i62, 1, _, 2, descr=<ArrayS 2>)
+        """)
diff --git a/pypy/module/struct/formatiterator.py b/pypy/module/struct/formatiterator.py
--- a/pypy/module/struct/formatiterator.py
+++ b/pypy/module/struct/formatiterator.py
@@ -2,7 +2,6 @@
                                       maxint, intmask)
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize
-from rpython.rlib.rstring import StringBuilder
 from rpython.rlib.rstruct.error import StructError
 from rpython.rlib.rstruct.formatiterator import FormatIterator
 
@@ -10,11 +9,15 @@
 
 
 class PackFormatIterator(FormatIterator):
-    def __init__(self, space, args_w, size):
+    def __init__(self, space, wbuf, args_w):
         self.space = space
         self.args_w = args_w
         self.args_index = 0
-        self.result = StringBuilder(size)
+        self.pos = 0
+        self.wbuf = wbuf
+
+    def advance(self, count):
+        self.pos += count
 
     # This *should* be always unroll safe, the only way to get here is by
     # unroll the interpret function, which means the fmt is const, and thus
@@ -31,8 +34,10 @@
 
     @jit.unroll_safe
     def align(self, mask):
-        pad = (-self.result.getlength()) & mask
-        self.result.append_multiple_char('\x00', pad)
+        pad = (-self.pos) & mask
+        for i in range(self.pos, self.pos+pad):
+            self.wbuf.setitem(i, '\x00')
+        self.advance(pad)
 
     def finished(self):
         if self.args_index != len(self.args_w):
@@ -135,15 +140,22 @@
         if value != self.length:
             raise StructError("unpack str size too long for format")
 
-    def read(self, count):
+    def can_advance(self, count):
         if self.strides:
             count = self.strides[0]
         end = self.pos + count
-        if end > self.length:
+        return end <= self.length
+
+    def advance(self, count):
+        if not self.can_advance(count):
             raise StructError("unpack str size too short for format")
-        s = self.buf.getslice(self.pos, end, 1, count)
-        self.pos = end
-        return s
+        self.pos += count
+
+    def read(self, count):
+        curpos = self.pos
+        end = curpos + count
+        self.advance(count) # raise if we are out of bound
+        return self.buf.getslice(curpos, end, 1, count)
 
     @specialize.argtype(1)
     def appendobj(self, value):
@@ -162,9 +174,8 @@
     def get_pos(self):
         return self.pos
 
-    def get_buffer_as_string_maybe(self):
-        string, pos = self.buf.as_str_and_offset_maybe()
-        return string, pos+self.pos
+    def get_buffer_and_pos(self):
+        return self.buf, self.pos
 
     def skip(self, count):
         # assumption: UnpackFormatIterator only iterates over
diff --git a/pypy/module/struct/interp_struct.py b/pypy/module/struct/interp_struct.py
--- a/pypy/module/struct/interp_struct.py
+++ b/pypy/module/struct/interp_struct.py
@@ -1,5 +1,6 @@
 from rpython.rlib import jit
 from rpython.rlib.buffer import SubBuffer
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct.error import StructError, StructOverflowError
 from rpython.rlib.rstruct.formatiterator import CalcSizeFormatIterator
 
@@ -48,18 +49,17 @@
 
 def _pack(space, format, args_w):
     """Return string containing values v1, v2, ... packed according to fmt."""
-    if jit.isconstant(format):
-        size = _calcsize(space, format)
-    else:
-        size = 8
-    fmtiter = PackFormatIterator(space, args_w, size)
+    size = _calcsize(space, format)
+    wbuf = MutableStringBuffer(size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
     try:
         fmtiter.interpret(format)
     except StructOverflowError as e:
         raise OperationError(space.w_OverflowError, space.newtext(e.msg))
     except StructError as e:
         raise OperationError(get_error(space), space.newtext(e.msg))
-    return fmtiter.result.build()
+    assert fmtiter.pos == wbuf.getlength(), 'missing .advance() or wrong calcsize()'
+    return wbuf.finish()
 
 
 def pack(space, w_format, args_w):
@@ -79,21 +79,27 @@
     format = text_or_bytes_w(space, w_format)
     return do_pack_into(space, format, w_buffer, offset, args_w)
 
-# XXX inefficient
 def do_pack_into(space, format, w_buffer, offset, args_w):
     """ Pack the values v1, v2, ... according to fmt.
 Write the packed bytes into the writable buffer buf starting at offset
     """
-    res = _pack(space, format, args_w)
+    size = _calcsize(space, format)
     buf = space.writebuf_w(w_buffer)
     if offset < 0:
         offset += buf.getlength()
-    size = len(res)
     if offset < 0 or (buf.getlength() - offset) < size:
         raise oefmt(get_error(space),
                     "pack_into requires a buffer of at least %d bytes",
                     size)
-    buf.setslice(offset, res)
+    #
+    wbuf = SubBuffer(buf, offset, size)
+    fmtiter = PackFormatIterator(space, wbuf, args_w)
+    try:
+        fmtiter.interpret(format)
+    except StructOverflowError as e:
+        raise OperationError(space.w_OverflowError, space.newtext(e.msg))
+    except StructError as e:
+        raise OperationError(get_error(space), space.newtext(e.msg))
 
 
 def _unpack(space, format, buf):
diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -591,7 +591,7 @@
 
 
 class AppTestFastPath(object):
-    spaceconfig = dict(usemodules=['struct', '__pypy__'])
+    spaceconfig = dict(usemodules=['array', 'struct', '__pypy__'])
 
     def setup_class(cls):
         from rpython.rlib.rstruct import standardfmttable
@@ -610,7 +610,43 @@
         from rpython.rlib.rstruct import standardfmttable
         standardfmttable.ALLOW_SLOWPATH = True
 
+    def test_unpack_simple(self):
+        buf = self.struct.pack("iii", 0, 42, 43)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
     def test_unpack_from(self):
         buf = self.struct.pack("iii", 0, 42, 43)
         offset = self.struct.calcsize("i")
         assert self.struct.unpack_from("ii", buf, offset) == (42, 43)
+
+    def test_unpack_bytearray(self):
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = bytearray(data)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_unpack_array(self):
+        import array
+        data = self.struct.pack("iii", 0, 42, 43)
+        buf = array.array('c', data)
+        assert self.struct.unpack("iii", buf) == (0, 42, 43)
+
+    def test_pack_into_bytearray(self):
+        expected = self.struct.pack("ii", 42, 43)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("ii", buf, 0, 42, 43)
+        assert buf == expected
+
+    def test_pack_into_bytearray_padding(self):
+        expected = self.struct.pack("xxi", 42)
+        buf = bytearray(len(expected))
+        self.struct.pack_into("xxi", buf, 0, 42)
+        assert buf == expected
+
+    def test_pack_into_bytearray_delete(self):
+        expected = self.struct.pack("i", 42)
+        # force W_BytearrayObject._delete_from_start
+        buf = bytearray(64)
+        del buf[:8]
+        self.struct.pack_into("i", buf, 0, 42)
+        buf = buf[:len(expected)]
+        assert buf == expected
diff --git a/pypy/objspace/std/bytearrayobject.py b/pypy/objspace/std/bytearrayobject.py
--- a/pypy/objspace/std/bytearrayobject.py
+++ b/pypy/objspace/std/bytearrayobject.py
@@ -8,10 +8,11 @@
 from rpython.rlib.debug import check_list_of_chars, check_nonneg
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
-        nonmoving_raw_ptr_for_resizable_list)
+                              nonmoving_raw_ptr_for_resizable_list)
 from rpython.rlib import jit
-from rpython.rlib.buffer import Buffer
-
+from rpython.rlib.buffer import (GCBuffer,
+                                 get_gc_data_for_list_of_chars,
+                                 get_gc_data_offset_for_list_of_chars)
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.objspace.std.bytesobject import makebytesdata_w, newbytesdata_w
@@ -1274,7 +1275,8 @@
         start += step
 
 
-class BytearrayBuffer(Buffer):
+ at GCBuffer.decorate
+class BytearrayBuffer(GCBuffer):
     _immutable_ = True
 
     def __init__(self, ba, readonly=False):
@@ -1304,7 +1306,7 @@
             if start != 0 or stop != len(data):
                 data = data[start:stop]
             return "".join(data)
-        return Buffer.getslice(self, start, stop, step, size)
+        return GCBuffer.getslice(self, start, stop, step, size)
 
     def setslice(self, start, string):
         # No bounds checks.
@@ -1319,6 +1321,16 @@
         p = rffi.ptradd(p, ba._offset)
         return p
 
+    @staticmethod
+    def _get_gc_data_offset():
+        return get_gc_data_offset_for_list_of_chars()
+
+    def _get_gc_data_extra_offset(self):
+        return self.ba._offset
+
+    def _get_gc_data(self):
+        return get_gc_data_for_list_of_chars(self.ba._data)
+
 
 @specialize.argtype(1)
 def _memcmp(selfvalue, buffer, length):
diff --git a/pypy/objspace/std/marshal_impl.py b/pypy/objspace/std/marshal_impl.py
--- a/pypy/objspace/std/marshal_impl.py
+++ b/pypy/objspace/std/marshal_impl.py
@@ -1,5 +1,6 @@
 from rpython.rlib.rarithmetic import LONG_BIT, r_longlong, r_uint
-from rpython.rlib.rstring import StringBuilder, assert_str0
+from rpython.rlib.rstring import assert_str0
+from rpython.rlib.mutbuffer import MutableStringBuffer
 from rpython.rlib.rstruct import ieee
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import objectmodel
@@ -213,9 +214,9 @@
 
 
 def pack_float(f):
-    result = StringBuilder(8)
-    ieee.pack_float(result, f, 8, False)
-    return result.build()
+    buf = MutableStringBuffer(8)
+    ieee.pack_float(buf, 0, f, 8, False)
+    return buf.finish()
 
 def unpack_float(s):
     return ieee.unpack_float(s, False)
@@ -408,7 +409,7 @@
     stacksize   = u.get_int()
     flags       = u.get_int()
     code        = space.bytes_w(u.get_w_obj())
-    consts_w    = _unmarshal_tuple_w(u)   
+    consts_w    = _unmarshal_tuple_w(u)
     names       = _unmarshal_strlist(u)
     varnames    = _unmarshal_strlist(u)
     freevars    = _unmarshal_strlist(u)
diff --git a/rpython/annotator/binaryop.py b/rpython/annotator/binaryop.py
--- a/rpython/annotator/binaryop.py
+++ b/rpython/annotator/binaryop.py
@@ -514,13 +514,13 @@
     ne = eq
 
     def lt((tup1, tup2)):
-        raise Exception("unsupported: (...) < (...)")
+        raise AnnotatorError("unsupported: (...) < (...)")
     def le((tup1, tup2)):
-        raise Exception("unsupported: (...) <= (...)")
+        raise AnnotatorError("unsupported: (...) <= (...)")
     def gt((tup1, tup2)):
-        raise Exception("unsupported: (...) > (...)")
+        raise AnnotatorError("unsupported: (...) > (...)")
     def ge((tup1, tup2)):
-        raise Exception("unsupported: (...) >= (...)")
+        raise AnnotatorError("unsupported: (...) >= (...)")
 
 
 class __extend__(pairtype(SomeDict, SomeDict)):
diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -15,7 +15,8 @@
     SomeDict, SomeBuiltin, SomePBC, SomeInteger, TLS, SomeUnicodeCodePoint,
     s_None, s_ImpossibleValue, SomeBool, SomeTuple, SomeException,
     SomeImpossibleValue, SomeUnicodeString, SomeList, HarmlesslyBlocked,
-    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty)
+    SomeWeakRef, SomeByteArray, SomeConstantType, SomeProperty,
+    AnnotatorError)
 from rpython.annotator.classdesc import ClassDef, ClassDesc
 from rpython.annotator.listdef import ListDef, ListItem
 from rpython.annotator.dictdef import DictDef
@@ -343,7 +344,7 @@
         elif x is None:
             return s_None
         else:
-            raise Exception("Don't know how to represent %r" % (x,))
+            raise AnnotatorError("Don't know how to represent %r" % (x,))
         result.const = x
         return result
 
@@ -363,7 +364,7 @@
                 result = self.newfuncdesc(pyobj)
             elif isinstance(pyobj, (type, types.ClassType)):
                 if pyobj is object:
-                    raise Exception("ClassDesc for object not supported")
+                    raise AnnotatorError("ClassDesc for object not supported")
                 if pyobj.__module__ == '__builtin__': # avoid making classdefs for builtin types
                     result = self.getfrozen(pyobj)
                 else:
@@ -400,7 +401,7 @@
                         msg = "object with a __call__ is not RPython"
                     else:
                         msg = "unexpected prebuilt constant"
-                    raise Exception("%s: %r" % (msg, pyobj))
+                    raise AnnotatorError("%s: %r" % (msg, pyobj))
                 result = self.getfrozen(pyobj)
             self.descs[obj_key] = result
             return result
@@ -589,7 +590,7 @@
         for name, value in dict.iteritems():
             if value is func:
                 return cls, name
-    raise Exception("could not match bound-method to attribute name: %r" % (boundmeth,))
+    raise AnnotatorError("could not match bound-method to attribute name: %r" % (boundmeth,))
 
 def ishashable(x):
     try:
diff --git a/rpython/annotator/classdesc.py b/rpython/annotator/classdesc.py
--- a/rpython/annotator/classdesc.py
+++ b/rpython/annotator/classdesc.py
@@ -550,8 +550,8 @@
                                         "with _mixin_: %r" % (cls,))
                 base = b1
         if mixins_before and mixins_after:
-            raise Exception("unsupported: class %r has mixin bases both"
-                            " before and after the regular base" % (self,))
+            raise AnnotatorError("unsupported: class %r has mixin bases both"
+                                 " before and after the regular base" % (self,))
         self.add_mixins(mixins_after, check_not_in=base)
         self.add_mixins(mixins_before)
         self.add_sources_for_class(cls)
@@ -569,8 +569,8 @@
                 attrs.update(decl)
             if self.basedesc is not None:
                 if self.basedesc.all_enforced_attrs is None:
-                    raise Exception("%r has slots or _attrs_, "
-                                    "but not its base class" % (cls,))
+                    raise AnnotatorError("%r has slots or _attrs_, "
+                                         "but not its base class" % (cls,))
                 attrs.update(self.basedesc.all_enforced_attrs)
             self.all_enforced_attrs = attrs
 
@@ -714,8 +714,8 @@
                 try:
                     args.fixedunpack(0)
                 except ValueError:
-                    raise Exception("default __init__ takes no argument"
-                                    " (class %s)" % (self.name,))
+                    raise AnnotatorError("default __init__ takes no argument"
+                                         " (class %s)" % (self.name,))
             elif self.pyobj is Exception:
                 # check explicitly against "raise Exception, x" where x
                 # is a low-level exception pointer
@@ -824,8 +824,8 @@
         # actual copy in the rtyper). Tested in rpython.rtyper.test.test_rlist,
         # test_immutable_list_out_of_instance.
         if self._detect_invalid_attrs and attr in self._detect_invalid_attrs:
-            raise Exception("field %r was migrated to %r from a subclass in "
-                            "which it was declared as _immutable_fields_" %
+            raise AnnotatorError("field %r was migrated to %r from a subclass in "
+                                 "which it was declared as _immutable_fields_" %
                             (attr, self.pyobj))
         search1 = '%s[*]' % (attr,)
         search2 = '%s?[*]' % (attr,)
@@ -858,7 +858,7 @@
             # call to a single class, look at the result annotation
             # in case it was specialized
             if not isinstance(s_result, SomeInstance):
-                raise Exception("calling a class didn't return an instance??")
+                raise AnnotatorError("calling a class didn't return an instance??")
             classdefs = [s_result.classdef]
         else:
             # call to multiple classes: specialization not supported
diff --git a/rpython/annotator/description.py b/rpython/annotator/description.py
--- a/rpython/annotator/description.py
+++ b/rpython/annotator/description.py
@@ -314,8 +314,8 @@
         enforceargs = getattr(self.pyobj, '_annenforceargs_', None)
         signature = getattr(self.pyobj, '_signature_', None)
         if enforceargs and signature:
-            raise Exception("%r: signature and enforceargs cannot both be "
-                            "used" % (self,))
+            raise AnnotatorError("%r: signature and enforceargs cannot both be "
+                                 "used" % (self,))
         if enforceargs:
             if not callable(enforceargs):
                 from rpython.annotator.signature import Sig
@@ -432,7 +432,7 @@
     def func_args(self, args):
         from rpython.annotator.model import SomeInstance
         if self.selfclassdef is None:
-            raise Exception("calling %r" % (self,))
+            raise AnnotatorError("calling %r" % (self,))
         s_instance = SomeInstance(self.selfclassdef, flags=self.flags)
         return args.prepend(s_instance)
 
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4584,7 +4584,7 @@
             return Ellipsis
         a = self.RPythonAnnotator()
         e = py.test.raises(Exception, a.build_types, f, [])
-        assert str(e.value) == "Don't know how to represent Ellipsis"
+        assert "Don't know how to represent Ellipsis" in str(e.value)
 
     def test_must_be_light_finalizer(self):
         from rpython.rlib import rgc
diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -185,8 +185,8 @@
         return SomeString()
 
     def id(self):
-        raise Exception("cannot use id() in RPython; "
-                        "see objectmodel.compute_xxx()")
+        raise AnnotatorError("cannot use id() in RPython; "
+                             "see objectmodel.compute_xxx()")
 
     def int(self):
         return SomeInteger()
@@ -421,7 +421,7 @@
     def setslice(self, s_start, s_stop, s_iterable):
         check_negative_slice(s_start, s_stop)
         if not isinstance(s_iterable, SomeList):
-            raise Exception("list[start:stop] = x: x must be a list")
+            raise AnnotatorError("list[start:stop] = x: x must be a list")
         self.listdef.mutate()
         self.listdef.agree(getbookkeeper(), s_iterable.listdef)
         self.listdef.resize()
diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -0,0 +1,9 @@
+from rpython.jit.backend.arm.test.support import JitARMMixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(JitARMMixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+    pass
+
diff --git a/rpython/jit/backend/llgraph/runner.py b/rpython/jit/backend/llgraph/runner.py
--- a/rpython/jit/backend/llgraph/runner.py
+++ b/rpython/jit/backend/llgraph/runner.py
@@ -716,16 +716,28 @@
         else:
             return self.bh_raw_load_i(struct, offset, descr)
 
+    def _get_int_type_from_size(self, size):
+        if   size == 1:
+            return rffi.UCHAR
+        elif size == 2:
+            return rffi.USHORT
+        elif size == 4:
+            return rffi.UINT
+        elif size == 8:
+            return rffi.ULONGLONG
+        elif size == -1:
+            return rffi.SIGNEDCHAR
+        elif size == -2:
+            return rffi.SHORT
+        elif size == -4:
+            return rffi.INT
+        elif size == -8:
+            return rffi.LONGLONG
+        else:
+            raise NotImplementedError(size)
+    
     def bh_gc_load_indexed_i(self, struct, index, scale, base_ofs, bytes):
-        if   bytes == 1: T = rffi.UCHAR
-        elif bytes == 2: T = rffi.USHORT
-        elif bytes == 4: T = rffi.UINT
-        elif bytes == 8: T = rffi.ULONGLONG
-        elif bytes == -1: T = rffi.SIGNEDCHAR
-        elif bytes == -2: T = rffi.SHORT
-        elif bytes == -4: T = rffi.INT
-        elif bytes == -8: T = rffi.LONGLONG
-        else: raise NotImplementedError(bytes)
+        T = self._get_int_type_from_size(bytes)
         x = llop.gc_load_indexed(T, struct, index, scale, base_ofs)
         return lltype.cast_primitive(lltype.Signed, x)
 
@@ -735,6 +747,30 @@
         return llop.gc_load_indexed(longlong.FLOATSTORAGE,
                                     struct, index, scale, base_ofs)
 
+    def bh_gc_store_indexed_i(self, struct, index, val, scale, base_ofs, bytes,
+                              descr):
+        T = self._get_int_type_from_size(bytes)
+        val = lltype.cast_primitive(T, val)
+        if descr.A.OF == lltype.SingleFloat:
+            val = longlong.int2singlefloat(val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+    def bh_gc_store_indexed_f(self, struct, index, val, scale, base_ofs, bytes,
+                              descr):
+        if bytes != 8:
+            raise Exception("gc_store_indexed_f is only for 'double'!")
+        val = longlong.getrealfloat(val)
+        llop.gc_store_indexed(lltype.Void, struct, index, val, scale, base_ofs)
+
+    def bh_gc_store_indexed(self, struct, index, val, scale, base_ofs, bytes,
+                            descr):
+        if descr.A.OF == lltype.Float:
+            self.bh_gc_store_indexed_f(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
+        else:
+            self.bh_gc_store_indexed_i(struct, index, val, scale, base_ofs,
+                                       bytes, descr)
+
     def bh_increment_debug_counter(self, addr):
         p = rffi.cast(rffi.CArrayPtr(lltype.Signed), addr)
         p[0] += 1
diff --git a/rpython/jit/backend/llsupport/llmodel.py b/rpython/jit/backend/llsupport/llmodel.py
--- a/rpython/jit/backend/llsupport/llmodel.py
+++ b/rpython/jit/backend/llsupport/llmodel.py
@@ -496,6 +496,7 @@
     @specialize.argtype(1)
     def write_float_at_mem(self, gcref, ofs, newvalue):
         llop.raw_store(lltype.Void, gcref, ofs, newvalue)
+    write_float_at_mem._annenforceargs_ = [None, None, None, longlong.r_float_storage]
 
     # ____________________________________________________________
 
@@ -754,6 +755,16 @@
         offset = base_ofs + scale * index
         return self.read_float_at_mem(addr, offset)
 
+    def bh_gc_store_indexed_i(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_int_at_mem(addr, offset, bytes, val)
+
+    def bh_gc_store_indexed_f(self, addr, index, val, scale, base_ofs, bytes,
+                              descr):
+        offset = base_ofs + scale * index
+        self.write_float_at_mem(addr, offset, val)
+
     def bh_new(self, sizedescr):
         return self.gc_ll_descr.gc_malloc(sizedescr)
 
diff --git a/rpython/jit/backend/x86/test/test_llop.py b/rpython/jit/backend/x86/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_llop.py
@@ -0,0 +1,14 @@
+from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
+from rpython.jit.metainterp.test.test_llop import TestLLOp as _TestLLOp
+
+
+class TestLLOp(Jit386Mixin, _TestLLOp):
+    # for the individual tests see
+    # ====> ../../../metainterp/test/test_llop.py
+
+    # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+    # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+    # symbolic as the offset. It is not a problem because it is tested anyway
+    # by the same test in test_metainterp.py
+    TEST_BLACKHOLE = False
+
diff --git a/rpython/jit/backend/x86/test/test_strstorage.py b/rpython/jit/backend/x86/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/backend/x86/test/test_strstorage.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from rpython.jit.backend.x86.test.test_basic import Jit386Mixin
-from rpython.jit.metainterp.test.test_strstorage import TestStrStorage as _TestStrStorage
-
-
-class TestStrStorage(Jit386Mixin, _TestStrStorage):
-    # for the individual tests see
-    # ====> ../../../metainterp/test/test_strstorage.py
-    pass
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -1131,6 +1131,26 @@
                               [op.args[0], op.args[1],
                                op.args[2], op.args[3], c_bytes], op.result)
 
+    def rewrite_op_gc_store_indexed(self, op):
+        T = op.args[2].concretetype
+        kind = getkind(T)[0]
+        assert kind != 'r'
+        descr = self.cpu.arraydescrof(rffi.CArray(T))
+        if (not isinstance(op.args[3], Constant) or
+            not isinstance(op.args[4], Constant)):
+            raise NotImplementedError("gc_store_indexed: 'scale' and 'base_ofs'"
+                                      " should be constants")
+        # According to the comment in resoperation.py, "itemsize is not signed
+        # (always > 0)", so we don't need the "bytes = -bytes" line which is
+        # in rewrite_op_gc_load_indexed
+        bytes = descr.get_item_size_in_bytes()
+        c_bytes = Constant(bytes, lltype.Signed)
+        return SpaceOperation('gc_store_indexed_%s' % kind,
+                              [op.args[0], op.args[1], op.args[2],
+                               op.args[3], op.args[4], c_bytes, descr], None)
+
+
+
     def _rewrite_equality(self, op, opname):
         arg0, arg1 = op.args
         if isinstance(arg0, Constant) and not arg0.value:
diff --git a/rpython/jit/metainterp/blackhole.py b/rpython/jit/metainterp/blackhole.py
--- a/rpython/jit/metainterp/blackhole.py
+++ b/rpython/jit/metainterp/blackhole.py
@@ -1478,6 +1478,18 @@
     def bhimpl_gc_load_indexed_f(cpu, addr, index, scale, base_ofs, bytes):
         return cpu.bh_gc_load_indexed_f(addr, index,scale,base_ofs, bytes)
 
+    @arguments("cpu", "r", "i", "i", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_i(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_i(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
+
+    @arguments("cpu", "r", "i", "f", "i", "i", "i", "d")
+    def bhimpl_gc_store_indexed_f(cpu, addr, index, val, scale, base_ofs, bytes,
+                                  arraydescr):
+        return cpu.bh_gc_store_indexed_f(addr, index, val, scale,base_ofs, bytes,
+                                         arraydescr)
+
     @arguments("r", "d", "d")
     def bhimpl_record_quasiimmut_field(struct, fielddescr, mutatefielddescr):
         pass
diff --git a/rpython/jit/metainterp/compile.py b/rpython/jit/metainterp/compile.py
--- a/rpython/jit/metainterp/compile.py
+++ b/rpython/jit/metainterp/compile.py
@@ -110,7 +110,7 @@
     """
     log_noopt = False
 
-    def __init__(self, trace, celltoken, state, runtime_boxes,
+    def __init__(self, trace, celltoken, state,
                  call_pure_results=None, enable_opts=None,
                  inline_short_preamble=True):
         self.trace = trace
@@ -119,8 +119,6 @@
         self.state = state
         self.call_pure_results = call_pure_results
         self.inline_short_preamble = inline_short_preamble
-        assert runtime_boxes is not None
-        self.runtime_boxes = runtime_boxes
 
     def optimize(self, metainterp_sd, jitdriver_sd, optimizations, unroll):
         from rpython.jit.metainterp.optimizeopt.unroll import UnrollOptimizer
@@ -128,11 +126,7 @@
         assert unroll # we should not be here if it's disabled
         opt = UnrollOptimizer(metainterp_sd, jitdriver_sd, optimizations)
         return opt.optimize_peeled_loop(self.trace, self.celltoken, self.state,
-            self.runtime_boxes, self.call_pure_results, self.inline_short_preamble)
-
-    def forget_optimization_info(self):
-        self.state.forget_optimization_info()
-        CompileData.forget_optimization_info(self)
+            self.call_pure_results, self.inline_short_preamble)
 
 def show_procedures(metainterp_sd, procedure=None, error=None):
     # debugging
@@ -298,7 +292,7 @@
     start_descr = TargetToken(jitcell_token,
                               original_jitcell_token=jitcell_token)
     jitcell_token.target_tokens = [start_descr]
-    loop_data = UnrolledLoopData(trace, jitcell_token, start_state, jumpargs,
+    loop_data = UnrolledLoopData(trace, jitcell_token, start_state,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -368,7 +362,7 @@
     history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
     enable_opts = jitdriver_sd.warmstate.enable_opts
     call_pure_results = metainterp.call_pure_results
-    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state, jumpargs,
+    loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
                                  call_pure_results=call_pure_results,
                                  enable_opts=enable_opts)
     try:
@@ -380,7 +374,6 @@
         history.cut(cut)
         history.record(rop.JUMP, jumpargs[:], None, descr=loop_jitcell_token)
         loop_data = UnrolledLoopData(trace, loop_jitcell_token, start_state,
-                                     jumpargs,
                                      call_pure_results=call_pure_results,
                                      enable_opts=enable_opts,
                                      inline_short_preamble=False)
@@ -525,7 +518,7 @@
     for item in lst:
         item.set_forwarded(None)
         # XXX we should really do it, but we need to remember the values
-        #     somehow for ContinueRunningNormally
+        #     somehoe for ContinueRunningNormally
         if reset_values:
             item.reset_value()
 
diff --git a/rpython/jit/metainterp/executor.py b/rpython/jit/metainterp/executor.py
--- a/rpython/jit/metainterp/executor.py
+++ b/rpython/jit/metainterp/executor.py
@@ -251,6 +251,25 @@
     else:
         return BoxInt(cpu.bh_raw_load_i(addr, offset, arraydescr))
 
+def do_gc_store_indexed(cpu, _, addrbox, indexbox, valuebox, scalebox,
+                        base_ofsbox, bytesbox, arraydescr):
+    addr = addrbox.getref_base()
+    index = indexbox.getint()
+    scale = scalebox.getint()
+    base_ofs = base_ofsbox.getint()
+    bytes = bytesbox.getint()
+    if arraydescr.is_array_of_pointers():
+        raise AssertionError("cannot store GC pointers in gc_store_indexed for now")
+    elif arraydescr.is_array_of_floats():
+        floatval = valuebox.getfloatstorage()
+        cpu.bh_gc_store_indexed_f(addr, index, floatval, scale, base_ofs, bytes,
+                                  arraydescr)
+    else:
+        intval = valuebox.getint()
+        cpu.bh_gc_store_indexed_i(addr, index, intval, scale, base_ofs, bytes,
+                                  arraydescr)
+
+
 def exec_new_with_vtable(cpu, descr):
     return cpu.bh_new_with_vtable(descr)
 
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -12,7 +12,7 @@
 from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.resoperation import rop, ResOperation, OpHelpers,\
      AbstractResOp, GuardResOp
-from rpython.rlib.objectmodel import we_are_translated, we_are_debug
+from rpython.rlib.objectmodel import we_are_translated
 from rpython.jit.metainterp.optimizeopt import info
         
 
@@ -173,7 +173,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getfield(descr, optheap)
-        if we_are_debug() and res:
+        if not we_are_translated() and res:
             if isinstance(opinfo, info.AbstractStructPtrInfo):
                 assert opinfo in self.cached_infos
         if isinstance(res, PreambleOp):
@@ -203,7 +203,7 @@
 
     def _getfield(self, opinfo, descr, optheap, true_force=True):
         res = opinfo.getitem(descr, self.index, optheap)
-        if we_are_debug() and res:
+        if not we_are_translated() and res:
             if isinstance(opinfo, info.ArrayPtrInfo):
                 assert opinfo in self.cached_infos
         if (isinstance(res, PreambleOp) and
diff --git a/rpython/jit/metainterp/optimizeopt/optimizer.py b/rpython/jit/metainterp/optimizeopt/optimizer.py
--- a/rpython/jit/metainterp/optimizeopt/optimizer.py
+++ b/rpython/jit/metainterp/optimizeopt/optimizer.py
@@ -24,20 +24,9 @@
 llhelper.CONST_NULLREF = llhelper.CONST_NULL
 REMOVED = AbstractResOp()
 
-def check_no_forwarding(lsts):
-    for lst in lsts:
-        for op in lst:
-            assert op.get_forwarded() is None
-
 class LoopInfo(object):
     label_op = None
 
-    def _check_no_forwarding(self):
-        pass
-
-    def forget_optimization_info(self):
-        pass
-
 class BasicLoopInfo(LoopInfo):
     def __init__(self, inputargs, quasi_immutable_deps, jump_op):
         self.inputargs = inputargs
@@ -567,8 +556,7 @@
         return (BasicLoopInfo(trace.inputargs, self.quasi_immutable_deps, last_op),
                 self._newoperations)
 
-    @staticmethod
-    def _clean_optimization_info(lst):
+    def _clean_optimization_info(self, lst):
         for op in lst:
             if op.get_forwarded() is not None:
                 op.set_forwarded(None)
diff --git a/rpython/jit/metainterp/optimizeopt/shortpreamble.py b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
--- a/rpython/jit/metainterp/optimizeopt/shortpreamble.py
+++ b/rpython/jit/metainterp/optimizeopt/shortpreamble.py
@@ -5,7 +5,6 @@
      rop, AbstractResOp, AbstractInputArg
 from rpython.jit.metainterp.history import Const, make_hashable_int,\
      TreeLoop
-from rpython.jit.metainterp.optimize import InvalidLoop
 from rpython.jit.metainterp.optimizeopt import info
 
 class PreambleOp(AbstractResOp):
@@ -19,7 +18,7 @@
     See force_op_from_preamble for details how the extra things are put.
     """
     op = None
-
+    
     def __init__(self, op, preamble_op, invented_name):
         self.op = op
         self.preamble_op = preamble_op
@@ -52,13 +51,7 @@
 class AbstractShortOp(object):
     """ An operation that is potentially produced by the short preamble
     """
-    res = None
-
-    def _check_no_forwarding(self):
-        assert self.res.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        self.res.clear_forwarded()
+    pass
 
 class HeapOp(AbstractShortOp):
     def __init__(self, res, getfield_op):
@@ -108,14 +101,6 @@
                                        descr=sop.getdescr())
         return ProducedShortOp(self, preamble_op)
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        assert self.getfield_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.getfield_op.clear_forwarded()
-
     def __repr__(self):
         return "HeapOp(%r)" % (self.res,)
 
@@ -208,16 +193,6 @@
                 l.append(pop)
         return l
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        self.one._check_no_forwarding()
-        self.two._check_no_forwarding()
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.one.forget_optimization_info()
-        self.two.forget_optimization_info()
-
     def repr(self, memo):
         return "CompoundOp(%s, %s, %s)" % (self.res.repr(memo),
                                            self.one.repr(memo),
@@ -228,7 +203,7 @@
 
 class ProducedShortOp(AbstractProducedShortOp):
     invented_name = False
-
+    
     def __init__(self, short_op, preamble_op):
         self.short_op = short_op
         self.preamble_op = preamble_op
@@ -240,14 +215,6 @@
     def repr(self, memo):
         return self.short_op.repr(memo)
 
-    def _check_no_forwarding(self):
-        self.short_op._check_no_forwarding()
-        assert self.preamble_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        self.short_op.forget_optimization_info()
-        self.preamble_op.clear_forwarded()
-
     def __repr__(self):
         return "%r -> %r" % (self.short_op, self.preamble_op)
 
@@ -268,14 +235,6 @@
     def repr(self, memo):
         return "INP(%s)" % (self.res.repr(memo),)
 
-    def _check_no_forwarding(self):
-        AbstractShortOp._check_no_forwarding(self)
-        assert self.preamble_op.get_forwarded() is None
-
-    def forget_optimization_info(self):
-        AbstractShortOp.forget_optimization_info(self)
-        self.preamble_op.clear_forwarded()
-
     def __repr__(self):
         return "INP(%r -> %r)" % (self.res, self.preamble_op)
 
@@ -495,23 +454,16 @@
         self.sb = sb
         self.extra_same_as = self.sb.extra_same_as
         self.target_token = target_token
-        self.build_inplace = False
 
     def setup(self, jump_args, short, label_args):
         self.jump_args = jump_args
         self.short = short
         self.label_args = label_args
-        self.build_inplace = True
 
     def add_preamble_op(self, preamble_op):
         """ Notice that we're actually using the preamble_op, add it to
         label and jump
         """
-        # Could this be considered a speculative error?
-        # This check should only fail when trying to jump to an existing trace
-        # by forcing portions of the virtualstate.
-        if not self.build_inplace:
-            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         op = preamble_op.op.get_box_replacement()
         if preamble_op.invented_name:
             self.extra_same_as.append(op)
@@ -519,8 +471,6 @@
         self.jump_args.append(preamble_op.preamble_op)
 
     def use_box(self, box, preamble_op, optimizer=None):
-        if not self.build_inplace:
-            raise InvalidLoop("Forcing boxes would modify an existing short preamble")
         jump_op = self.short.pop()
         AbstractShortPreambleBuilder.use_box(self, box, preamble_op, optimizer)
         self.short.append(jump_op)
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_util.py b/rpython/jit/metainterp/optimizeopt/test/test_util.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_util.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_util.py
@@ -577,7 +577,6 @@
         #
         compile_data.enable_opts = self.enable_opts
         state = optimize_trace(metainterp_sd, None, compile_data)
-        state[0]._check_no_forwarding()
         return state
 
     def _convert_call_pure_results(self, d):
@@ -626,7 +625,7 @@
         start_state, preamble_ops = self._do_optimize_loop(preamble_data)
         preamble_data.forget_optimization_info()
         loop_data = compile.UnrolledLoopData(preamble_data.trace,
-            celltoken, start_state, runtime_boxes, call_pure_results)
+            celltoken, start_state, call_pure_results)
         loop_info, ops = self._do_optimize_loop(loop_data)
         preamble = TreeLoop('preamble')
         preamble.inputargs = start_state.renamed_inputargs
diff --git a/rpython/jit/metainterp/optimizeopt/unroll.py b/rpython/jit/metainterp/optimizeopt/unroll.py
--- a/rpython/jit/metainterp/optimizeopt/unroll.py
+++ b/rpython/jit/metainterp/optimizeopt/unroll.py
@@ -6,7 +6,7 @@
 from rpython.jit.metainterp.optimizeopt import info, intutils
 from rpython.jit.metainterp.optimize import InvalidLoop, SpeculativeError
 from rpython.jit.metainterp.optimizeopt.optimizer import Optimizer,\
-     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo, check_no_forwarding
+     Optimization, LoopInfo, MININT, MAXINT, BasicLoopInfo
 from rpython.jit.metainterp.optimizeopt.vstring import StrPtrInfo
 from rpython.jit.metainterp.optimizeopt.virtualstate import (
     VirtualStateConstructor, VirtualStatesCantMatch)
@@ -35,7 +35,7 @@
 
     def setinfo_from_preamble_list(self, lst, infos):
         for item in lst:
-            if item is None or isinstance(item, Const):
+            if item is None:
                 continue
             i = infos.get(item, None)
             if i is not None:
@@ -99,6 +99,7 @@
         elif isinstance(preamble_info, info.FloatConstInfo):
             op.set_forwarded(preamble_info._const)
 
+
 class UnrollOptimizer(Optimization):
     """Unroll the loop into two iterations. The first one will
     become the preamble or entry bridge (don't think there is a
@@ -116,22 +117,26 @@
         return modifier.get_virtual_state(args)
 
     def _check_no_forwarding(self, lsts, check_newops=True):
-        check_no_forwarding(lsts)
+        for lst in lsts:
+            for op in lst:
+                assert op.get_forwarded() is None
         if check_newops:
             assert not self.optimizer._newoperations
 
+
     def optimize_preamble(self, trace, runtime_boxes, call_pure_results, memo):
         info, newops = self.optimizer.propagate_all_forward(
             trace.get_iter(), call_pure_results, flush=False)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, memo)
+                                           info.inputargs,
+                                           runtime_boxes, memo)
         exported_state.quasi_immutable_deps = info.quasi_immutable_deps
         # we need to absolutely make sure that we've cleaned up all
         # the optimization info
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
 
-    def optimize_peeled_loop(self, trace, celltoken, state, runtime_boxes,
+    def optimize_peeled_loop(self, trace, celltoken, state,
                              call_pure_results, inline_short_preamble=True):
         trace = trace.get_iter()
         try:
@@ -183,7 +188,7 @@
 
         try:
             new_virtual_state = self.jump_to_existing_trace(
-                    end_jump, label_op, runtime_boxes, force_boxes=False)
+                    end_jump, label_op, state.runtime_boxes, force_boxes=False)
         except InvalidLoop:
             # inlining short preamble failed, jump to preamble
             self.jump_to_preamble(celltoken, end_jump, info)
@@ -196,7 +201,7 @@
             # to the preamble.
             try:
                 new_virtual_state = self.jump_to_existing_trace(
-                        end_jump, label_op, runtime_boxes, force_boxes=True)
+                        end_jump, label_op, state.runtime_boxes, force_boxes=True)
             except InvalidLoop:
                 pass
 
@@ -279,7 +284,8 @@
             debug_print("Retrace count reached, jumping to preamble")
             return self.jump_to_preamble(cell_token, jump_op, info)
         exported_state = self.export_state(info.jump_op.getarglist(),
-                                           info.inputargs, box_names_memo)
+                                           info.inputargs, runtime_boxes,
+                                           box_names_memo)
         exported_state.quasi_immutable_deps = self.optimizer.quasi_immutable_deps
         self.optimizer._clean_optimization_info(self.optimizer._newoperations)
         return exported_state, self.optimizer._newoperations
@@ -443,7 +449,8 @@
                 continue
             self._expand_info(item, infos)
 
-    def export_state(self, original_label_args, renamed_inputargs, memo):
+    def export_state(self, original_label_args, renamed_inputargs,
+                     runtime_boxes, memo):
         end_args = [self.optimizer.force_box_for_end_of_preamble(a)
                     for a in original_label_args]
         self.optimizer.flush()
@@ -464,17 +471,16 @@
             op = produced_op.short_op.res
             if not isinstance(op, Const):
                 self._expand_info(op, infos)
+        self.optimizer._clean_optimization_info(end_args)
         return ExportedState(label_args, end_args, virtual_state, infos,
                              short_boxes, renamed_inputargs,
-                             short_inputargs, memo)
+                             short_inputargs, runtime_boxes, memo)
 
     def import_state(self, targetargs, exported_state):
         # the mapping between input args (from old label) and what we need
         # to actually emit. Update the info
         assert (len(exported_state.next_iteration_args) ==
                 len(targetargs))
-        self._check_no_forwarding([targetargs])
-        exported_state._check_no_forwarding()
         for i, target in enumerate(exported_state.next_iteration_args):
             source = targetargs[i]
             assert source is not target
@@ -530,11 +536,13 @@
     * renamed_inputargs - the start label arguments in optimized version
     * short_inputargs - the renamed inputargs for short preamble
     * quasi_immutable_deps - for tracking quasi immutables
+    * runtime_boxes - runtime values for boxes, necessary when generating
+                      guards to jump to
     """
 
     def __init__(self, end_args, next_iteration_args, virtual_state,
                  exported_infos, short_boxes, renamed_inputargs,
-                 short_inputargs, memo):
+                 short_inputargs, runtime_boxes, memo):
         self.end_args = end_args
         self.next_iteration_args = next_iteration_args
         self.virtual_state = virtual_state
@@ -542,8 +550,8 @@
         self.short_boxes = short_boxes
         self.renamed_inputargs = renamed_inputargs
         self.short_inputargs = short_inputargs
+        self.runtime_boxes = runtime_boxes
         self.dump(memo)
-        self.forget_optimization_info()
 
     def dump(self, memo):
         if have_debug_prints():
@@ -553,35 +561,5 @@
                 debug_print("  " + box.repr(memo))
             debug_stop("jit-log-exported-state")
 
-    def _check_no_forwarding(self):
-        """ Ensures that no optimization state is attached to relevant operations
-        before importing anything. """
-        # Some of these may be redunant
-        check_no_forwarding([
-            self.end_args,
-            self.next_iteration_args,
-            self.renamed_inputargs,
-            self.short_inputargs,
-            self.exported_infos.keys()])
-        for box in self.short_boxes:
-            box._check_no_forwarding()
-
-    def forget_optimization_info(self):
-        """ Clean up optimization info on all operations stored in the ExportedState.
-
-        This function needs to be called when exporting the optimizer state to
-        prevent leaking of optimization information between invocations of the
-        optimizer.
-
-        That includes cleaning up in the event that optimize_peeled_loop() fails
-        with an InvalidLoop exception, as optimize_peeled_loop() mutates the
-        contents of ExportedState.
-        """
-        Optimizer._clean_optimization_info(self.renamed_inputargs)
-        for box in self.exported_infos.iterkeys():
-            box.clear_forwarded()
-        for box in self.short_boxes:
-            box.forget_optimization_info()
-
     def final(self):
         return False
diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py
--- a/rpython/jit/metainterp/pyjitpl.py
+++ b/rpython/jit/metainterp/pyjitpl.py
@@ -827,6 +827,21 @@
                             self._remove_symbolics(scalebox),
                             self._remove_symbolics(baseofsbox), bytesbox)
 
+    @arguments("box", "box", "box", "box", "box", "box", "descr")
+    def _opimpl_gc_store_indexed(self, addrbox, indexbox, valuebox,
+                                 scalebox, baseofsbox, bytesbox,
+                                 arraydescr):
+        return self.execute_with_descr(rop.GC_STORE_INDEXED,
+                                       arraydescr,
+                                       addrbox,
+                                       indexbox,
+                                       valuebox,
+                                       self._remove_symbolics(scalebox),
+                                       self._remove_symbolics(baseofsbox),
+                                       bytesbox)
+    opimpl_gc_store_indexed_i = _opimpl_gc_store_indexed
+    opimpl_gc_store_indexed_f = _opimpl_gc_store_indexed
+
     @arguments("box")
     def opimpl_hint_force_virtualizable(self, box):
         self.metainterp.gen_store_back_in_vable(box)
@@ -2024,8 +2039,6 @@
         self.aborted_tracing_greenkey = None
 
     def retrace_needed(self, trace, exported_state):
-        if not we_are_translated():
-            exported_state._check_no_forwarding()
         self.partial_trace = trace
         self.retracing_from = self.potential_retrace_position
         self.exported_state = exported_state
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -52,10 +52,6 @@
         llop.debug_print(lltype.Void, "setting forwarded on:", self.__class__.__name__)
         raise SettingForwardedOnAbstractValue()
 
-    def clear_forwarded(self):
-        if self.get_forwarded() is not None:
-            self.set_forwarded(None)
-
     @specialize.arg(1)
     def get_box_replacement(op, not_const=False):
         # Read the chain "op, op._forwarded, op._forwarded._forwarded..."
diff --git a/rpython/jit/metainterp/test/support.py b/rpython/jit/metainterp/test/support.py
--- a/rpython/jit/metainterp/test/support.py
+++ b/rpython/jit/metainterp/test/support.py
@@ -19,7 +19,8 @@
                   supports_floats=True,
                   supports_longlong=False,
                   supports_singlefloats=False,
-                  translationoptions={}, **kwds):
+                  translationoptions={},
+                  backendopt_inline_threshold=0, **kwds):
     from rpython.jit.codewriter import support
 
     class FakeJitCell(object):
@@ -59,7 +60,7 @@
         FakeWarmRunnerState.enable_opts = {}
 
     func._jit_unroll_safe_ = True
-    rtyper = support.annotate(func, values,
+    rtyper = support.annotate(func, values, inline=backendopt_inline_threshold,
                               translationoptions=translationoptions)
     graphs = rtyper.annotator.translator.graphs
     testself.all_graphs = graphs
diff --git a/rpython/jit/metainterp/test/test_llop.py b/rpython/jit/metainterp/test/test_llop.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/metainterp/test/test_llop.py
@@ -0,0 +1,72 @@
+import py
+import sys
+import struct
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.test.test_llop import (BaseLLOpTest, str_gc_load,
+                                           newlist_and_gc_store)
+from rpython.jit.codewriter import longlong
+from rpython.jit.metainterp.history import getkind
+from rpython.jit.metainterp.test.support import LLJitMixin
+
+
+class TestLLOp(BaseLLOpTest, LLJitMixin):
+
+    # for the individual tests see
+    # ====> ../../../rtyper/test/test_llop.py
+    TEST_BLACKHOLE = True
+
+    def gc_load_from_string(self, TYPE, buf, offset):
+        def f(offset):
+            return str_gc_load(TYPE, buf, offset)
+        res = self.interp_operations(f, [offset], supports_singlefloats=True)
+        #
+        kind = getkind(TYPE)[0] # 'i' or 'f'
+        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
+                                       'finish': 1})
+        #
+        if TYPE == lltype.SingleFloat:
+            # interp_operations returns the int version of r_singlefloat, but
+            # our tests expects to receive an r_singlefloat: let's convert it
+            # back!
+            return longlong.int2singlefloat(res)
+        return res
+
+    def newlist_and_gc_store(self, TYPE, value, expected):
+        def f(value):
+            lst = newlist_and_gc_store(TYPE, value)
+            got = ''.join(lst)
+            if got != expected:
+                # I'm not sure why, but if I use an assert, the test doesn't fail
+                raise ValueError('got != expected')
+            return len(got)
+        #
+        if self.TEST_BLACKHOLE:
+            # we pass a big inline_threshold to ensure that
+            # newlist_and_gc_store is inlined, else the blackhole does not see
+            # (and thus we do not test!) the llop.gc_store_indexed
+            threshold = 33
+        else:
+            threshold = 0
+        return self.interp_operations(f, [value], supports_singlefloats=True,
+                                      backendopt_inline_threshold=threshold)
+
+
+    def test_force_virtual_str_storage(self):
+        byteorder = sys.byteorder
+        size = rffi.sizeof(lltype.Signed)
+        def f(val):
+            if byteorder == 'little':
+                x = chr(val) + '\x00'*(size-1)
+            else:
+                x = '\x00'*(size-1) + chr(val)
+            return str_gc_load(lltype.Signed, x, 0)
+        res = self.interp_operations(f, [42], supports_singlefloats=True)
+        assert res == 42
+        self.check_operations_history({
+            'newstr': 1,              # str forcing
+            'strsetitem': 1,          # str forcing
+            'call_pure_r': 1,         # str forcing (copystrcontent)
+            'guard_no_exception': 1,  # str forcing
+            'gc_load_indexed_i': 1,   # str_storage_getitem
+            'finish': 1
+            })
diff --git a/rpython/jit/metainterp/test/test_strstorage.py b/rpython/jit/metainterp/test/test_strstorage.py
deleted file mode 100644
--- a/rpython/jit/metainterp/test/test_strstorage.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import py
-import sys
-import struct
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib.strstorage import str_storage_getitem
-from rpython.rlib.test.test_strstorage import BaseStrStorageTest
-from rpython.jit.codewriter import longlong
-from rpython.jit.metainterp.history import getkind
-from rpython.jit.metainterp.test.support import LLJitMixin
-
-class TestStrStorage(BaseStrStorageTest, LLJitMixin):
-
-    # for the individual tests see
-    # ====> ../../../rlib/test/test_strstorage.py
-
-    def str_storage_getitem(self, TYPE, buf, offset):
-        def f():
-            return str_storage_getitem(TYPE, buf, offset)
-        res = self.interp_operations(f, [], supports_singlefloats=True)
-        #
-        kind = getkind(TYPE)[0] # 'i' or 'f'
-        self.check_operations_history({'gc_load_indexed_%s' % kind: 1,
-                                       'finish': 1})
-        #
-        if TYPE == lltype.SingleFloat:
-            # interp_operations returns the int version of r_singlefloat, but
-            # our tests expects to receive an r_singlefloat: let's convert it
-            # back!
-            return longlong.int2singlefloat(res)
-        return res
-
-    #def str_storage_supported(self, TYPE):
-    #    py.test.skip('this is not a JIT test')
-
-    def test_force_virtual_str_storage(self):
-        byteorder = sys.byteorder
-        size = rffi.sizeof(lltype.Signed)
-        def f(val):
-            if byteorder == 'little':
-                x = chr(val) + '\x00'*(size-1)
-            else:
-                x = '\x00'*(size-1) + chr(val)
-            return str_storage_getitem(lltype.Signed, x, 0)
-        res = self.interp_operations(f, [42], supports_singlefloats=True)
-        assert res == 42
-        self.check_operations_history({
-            'newstr': 1,              # str forcing
-            'strsetitem': 1,          # str forcing
-            'call_pure_r': 1,         # str forcing (copystrcontent)
-            'guard_no_exception': 1,  # str forcing
-            'gc_load_indexed_i': 1,   # str_storage_getitem
-            'finish': 1
-            })
diff --git a/rpython/rlib/buffer.py b/rpython/rlib/buffer.py
--- a/rpython/rlib/buffer.py
+++ b/rpython/rlib/buffer.py
@@ -1,13 +1,55 @@
 """
 Buffer protocol support.
 """
-from rpython.rlib.rgc import (
-    nonmoving_raw_ptr_for_resizable_list, resizable_list_supporting_raw_ptr)
+from rpython.rtyper.lltypesystem import lltype, llmemory
+from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.lltypesystem.rstr import STR
+from rpython.rtyper.lltypesystem.rlist import LIST_OF
+from rpython.rtyper.annlowlevel import llstr
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib import jit
+from rpython.rlib.rgc import (resizable_list_supporting_raw_ptr,
+                              nonmoving_raw_ptr_for_resizable_list,
+                              ll_for_resizable_list)
 from rpython.rlib.signature import signature
 from rpython.rlib import types
+from rpython.rlib import rawstorage
+
+ALLOW_UNALIGNED_ACCESS = rawstorage.misaligned_is_fine
+
+ at specialize.ll()
+def is_alignment_correct(TYPE, index):
+    if ALLOW_UNALIGNED_ACCESS:
+        return True
+    try:
+        rawstorage._check_alignment(TYPE, index)
+    except rawstorage.AlignmentError:
+        return False
+    else:
+        return True
+
+
+class CannotRead(Exception):
+    """
+    Exception raised by Buffer.typed_read in case it is not possible to
+    accomplish the request. This might be because it is not supported by the
+    specific type of buffer, or because of alignment issues.
+    """
+
+class CannotWrite(Exception):
+    """
+    Raised by Buffer.typed_write in case it is not possible to accomplish the
+    request
+    """
 
 class Buffer(object):
-    """Base class for buffers of bytes"""
+    """
+    Base class for buffers of bytes.
+
+    Most probably, you do NOT want to use this as a lone base class, but
+    either inherit from RawBuffer or GCBuffer, so that you automatically get
+    the proper implementation of typed_read and typed_write.
+    """
     _attrs_ = ['readonly']
     _immutable_ = True
 
@@ -25,14 +67,6 @@
         # May be overridden.
         return self.getslice(0, self.getlength(), 1, self.getlength())
 
-    def as_str_and_offset_maybe(self):
-        """
-        If the buffer is backed by a string, return a pair (string, offset),
-        where offset is the offset inside the string where the buffer start.
-        Else, return (None, 0).
-        """
-        return None, 0
-
     def getitem(self, index):
         "Returns the index'th character in the buffer."
         raise NotImplementedError   # Must be overriden.  No bounds checks.
@@ -60,7 +94,141 @@
         for i in range(len(string)):
             self.setitem(start + i, string[i])
 
-class ByteBuffer(Buffer):
+    @jit.look_inside_iff(lambda self, index, count:
+                         jit.isconstant(count) and count <= 8)
+    def setzeros(self, index, count):
+        for i in range(index, index+count):

From pypy.commits at gmail.com  Wed May 24 13:06:20 2017
From: pypy.commits at gmail.com (stefanor)
Date: Wed, 24 May 2017 10:06:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Everything in lib_pypy should at least be
 compatible with py3k syntax
Message-ID: <5925bd8c.c888df0a.9420d.0ec1@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: py3.5
Changeset: r91409:ad5a0fedc353
Date: 2017-05-24 10:05 -0700
http://bitbucket.org/pypy/pypy/changeset/ad5a0fedc353/

Log:	Everything in lib_pypy should at least be compatible with py3k
	syntax

diff --git a/lib_pypy/pyrepl/keymaps.py b/lib_pypy/pyrepl/keymaps.py
--- a/lib_pypy/pyrepl/keymaps.py
+++ b/lib_pypy/pyrepl/keymaps.py
@@ -62,7 +62,7 @@
      (r'\M-\n', 'self-insert'),
      (r'\<backslash>', 'self-insert')] + \
     [(c, 'self-insert')
-     for c in map(chr, range(32, 127)) if c <> '\\'] + \
+     for c in map(chr, range(32, 127)) if c != '\\'] + \
     [(c, 'self-insert')
      for c in map(chr, range(128, 256)) if c.isalpha()] + \
     [(r'\<up>', 'up'),
@@ -101,7 +101,7 @@
     
 reader_vi_insert_keymap = tuple(
     [(c, 'self-insert')
-     for c in map(chr, range(32, 127)) if c <> '\\'] + \
+     for c in map(chr, range(32, 127)) if c != '\\'] + \
     [(c, 'self-insert')
      for c in map(chr, range(128, 256)) if c.isalpha()] + \
     [(r'\C-d', 'delete'),
diff --git a/lib_pypy/pyrepl/pygame_console.py b/lib_pypy/pyrepl/pygame_console.py
--- a/lib_pypy/pyrepl/pygame_console.py
+++ b/lib_pypy/pyrepl/pygame_console.py
@@ -130,7 +130,7 @@
         s.fill(c, [0, 600 - bmargin, 800, bmargin])
         s.fill(c, [800 - rmargin, 0, lmargin, 600])
 
-    def refresh(self, screen, (cx, cy)):
+    def refresh(self, screen, cxy):
         self.screen = screen
         self.pygame_screen.fill(colors.bg,
                                 [0, tmargin + self.cur_top + self.scroll,
@@ -139,6 +139,7 @@
 
         line_top = self.cur_top
         width, height = self.fontsize
+        cx, cy = cxy
         self.cxy = (cx, cy)
         cp = self.char_pos(cx, cy)
         if cp[1] < tmargin:
@@ -282,7 +283,7 @@
 
     def forgetinput(self):
         """Forget all pending, but not yet processed input."""
-        while pygame.event.poll().type <> NOEVENT:
+        while pygame.event.poll().type != NOEVENT:
             pass
     
     def getpending(self):
@@ -299,7 +300,7 @@
 
     def wait(self):
         """Wait for an event."""
-        raise Exception, "erp!"
+        raise Exception("erp!")
 
     def repaint(self):
         # perhaps we should consolidate grobs?
diff --git a/lib_pypy/pyrepl/pygame_keymap.py b/lib_pypy/pyrepl/pygame_keymap.py
--- a/lib_pypy/pyrepl/pygame_keymap.py
+++ b/lib_pypy/pyrepl/pygame_keymap.py
@@ -90,22 +90,22 @@
                 s += 2
             elif c == "c":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\C must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if ctrl:
-                    raise KeySpecError, "doubled \\C- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\C- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 ctrl = 1
                 s += 3
             elif c == "m":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\M must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if meta:
-                    raise KeySpecError, "doubled \\M- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\M- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 meta = 1
                 s += 3
             elif c.isdigit():
@@ -119,22 +119,22 @@
             elif c == '<':
                 t = key.find('>', s)
                 if t == -1:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unterminated \\< starting at char %d of %s"%(
-                        s + 1, repr(key))
+                        s + 1, repr(key)))
                 try:
                     ret = _keynames[key[s+2:t].lower()]
                     s = t + 1
                 except KeyError:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unrecognised keyname `%s' at char %d of %s"%(
-                        key[s+2:t], s + 2, repr(key))
+                        key[s+2:t], s + 2, repr(key)))
                 if ret is None:
                     return None, s
             else:
-                raise KeySpecError, \
+                raise KeySpecError(
                           "unknown backslash escape %s at char %d of %s"%(
-                    `c`, s + 2, repr(key))
+                    repr(c), s + 2, repr(key)))
         else:
             if ctrl:
                 ret = chr(ord(key[s]) & 0x1f)   # curses.ascii.ctrl()
@@ -160,9 +160,9 @@
         r.setdefault(key[0], {})[key[1:]] = value
     for key, value in r.items():
         if value.has_key(()):
-            if len(value) <> 1:
-                raise KeySpecError, \
-                          "key definitions for %s clash"%(value.values(),)
+            if len(value) != 1:
+                raise KeySpecError(
+                          "key definitions for %s clash"%(value.values(),))
             else:
                 r[key] = value[()]
         else:
@@ -202,7 +202,7 @@
         return ''
     name, s = keyname(keyseq)
     if name:
-        if name <> 'escape' or s == len(keyseq):
+        if name != 'escape' or s == len(keyseq):
             return '\\<' + name + '>' + unparse_key(keyseq[s:])
         else:
             return '\\M-' + unparse_key(keyseq[1:])
@@ -226,7 +226,7 @@
         return []
     name, s = keyname(keyseq)
     if name:
-        if name <> 'escape' or s == len(keyseq):
+        if name != 'escape' or s == len(keyseq):
             return [name] + _unparse_keyf(keyseq[s:])
         else:
             rest = _unparse_keyf(keyseq[1:])

From pypy.commits at gmail.com  Thu May 25 12:57:01 2017
From: pypy.commits at gmail.com (antocuni)
Date: Thu, 25 May 2017 09:57:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: don't test the blackhole path in ARM
 tests, for the very same reason as we do in x86 ones
Message-ID: <59270cdd.4297df0a.90105.e5f2@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91410:190cd7ec3d9a
Date: 2017-05-25 18:56 +0200
http://bitbucket.org/pypy/pypy/changeset/190cd7ec3d9a/

Log:	don't test the blackhole path in ARM tests, for the very same reason
	as we do in x86 ones

diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
--- a/rpython/jit/backend/arm/test/test_llop.py
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -5,5 +5,9 @@
 class TestLLOp(JitARMMixin, _TestLLOp):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_llop.py
-    pass
 
+    # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+    # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+    # symbolic as the offset. It is not a problem because it is tested anyway
+    # by the same test in test_metainterp.py
+    TEST_BLACKHOLE = False

From pypy.commits at gmail.com  Thu May 25 15:22:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 25 May 2017 12:22:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix space.bufferstr_w() returning a
 wrong value for numpy arrays and add an indirect test for this
Message-ID: <59272f0b.c888df0a.9420d.55fd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91411:3855d4baa148
Date: 2017-05-25 20:20 +0100
http://bitbucket.org/pypy/pypy/changeset/3855d4baa148/

Log:	Fix space.bufferstr_w() returning a wrong value for numpy arrays and
	add an indirect test for this

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -389,6 +389,9 @@
         for i in range(len(string)):
             self.ptr[start + i] = string[i]
 
+    def as_str(self):
+        return CBuffer(self).as_str()
+
     def as_readbuf(self):
         return CBuffer(self)
 
diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2011,9 +2011,19 @@
 static int
 array_getbuffer(PyObject* obj, Py_buffer* view, int flags)
 {
+    int ret;
     arrayobject* self = (arrayobject*)obj;
-    return PyBuffer_FillInfo(view, obj, self->ob_item,
+    ret = PyBuffer_FillInfo(view, obj, self->ob_item,
             Py_SIZE(self)*self->ob_descr->itemsize, 0, flags);
+    if (ret < 0) {
+        return ret;
+    }
+    if ((flags & PyBUF_ND) != PyBUF_ND) {
+        // numpy effectively does this
+        view->ndim = 0;
+        view->shape = NULL;
+    }
+    return ret;
 }
 
 static long releasebuffer_cnt = 0;
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -34,27 +34,27 @@
 
     def test_index(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
+        arr = module.array('i', [1, 2, 3, 4])
         assert arr[3] == 4
         raises(IndexError, arr.__getitem__, 10)
         del arr[2]
-        assert arr.tolist() == [1,2,4]
+        assert arr.tolist() == [1, 2, 4]
         arr[2] = 99
-        assert arr.tolist() == [1,2,99]
+        assert arr.tolist() == [1, 2, 99]
 
     def test_slice_get(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
-        assert arr[:].tolist() == [1,2,3,4]
-        assert arr[1:].tolist() == [2,3,4]
-        assert arr[:2].tolist() == [1,2]
-        assert arr[1:3].tolist() == [2,3]
+        arr = module.array('i', [1, 2, 3, 4])
+        assert arr[:].tolist() == [1, 2, 3, 4]
+        assert arr[1:].tolist() == [2, 3, 4]
+        assert arr[:2].tolist() == [1, 2]
+        assert arr[1:3].tolist() == [2, 3]
 
     def test_slice_object(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
-        assert arr[slice(1,3)].tolist() == [2,3]
-        arr[slice(1,3)] = module.array('i', [21, 22, 23])
+        arr = module.array('i', [1, 2, 3, 4])
+        assert arr[slice(1, 3)].tolist() == [2,3]
+        arr[slice(1, 3)] = module.array('i', [21, 22, 23])
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
@@ -63,20 +63,16 @@
     def test_buffer(self):
         import sys
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
+        arr = module.array('i', [1, 2, 3, 4])
         buf = buffer(arr)
         exc = raises(TypeError, "buf[1] = '1'")
         assert str(exc.value) == "buffer is read-only"
         if sys.byteorder == 'big':
-            assert str(buf) == ('\0\0\0\x01'
-                                '\0\0\0\x02'
-                                '\0\0\0\x03'
-                                '\0\0\0\x04')
+            expected = '\0\0\0\x01' '\0\0\0\x02' '\0\0\0\x03' '\0\0\0\x04'
         else:
-            assert str(buf) == ('\x01\0\0\0'
-                                '\x02\0\0\0'
-                                '\x03\0\0\0'
-                                '\x04\0\0\0')
+            expected = '\x01\0\0\0' '\x02\0\0\0' '\x03\0\0\0' '\x04\0\0\0'
+        assert str(buf) == expected
+        assert str(buffer('') + arr) == expected
 
     def test_releasebuffer(self):
         module = self.import_module(name='array')

From pypy.commits at gmail.com  Thu May 25 21:08:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 25 May 2017 18:08:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5927800a.1887df0a.d084e.9644@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91412:a76798b64382
Date: 2017-05-26 02:08 +0100
http://bitbucket.org/pypy/pypy/changeset/a76798b64382/

Log:	hg merge default

diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -278,11 +278,14 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if (config.translation.profopt is not None
-                    and not config.translation.noprofopt):
-                raise Exception("Cannot use the --profopt option "
-                                "when --shared is on (it is by default). "
-                                "See issue #2398.")
+
+        # if both profopt and profoptpath are specified then we keep them as they are with no other changes
+        if config.translation.profopt:
+            if config.translation.profoptargs is None:
+                config.translation.profoptargs = "$(RPYDIR)/../lib-python/2.7/test/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true"
+        elif config.translation.profoptargs is not None:
+            raise Exception("Cannot use --profoptargs without specifying --profopt as well")
+
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -830,6 +830,11 @@
     executable = sys.pypy_find_executable(executable)
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
+        for lib_path in sys.path:
+            stdlib_path = sys.pypy_find_stdlib(lib_path)
+            if stdlib_path is not None:
+                break
+    if stdlib_path is None:
         initstdio()
         print(STDLIB_WARNING, file=sys.stderr)
     else:
diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -390,6 +390,9 @@
         for i in range(len(string)):
             self.ptr[start + i] = string[i]
 
+    def as_str(self):
+        return CBuffer(self).as_str()
+
     def as_readbuf(self):
         return CBuffer(self)
 
diff --git a/pypy/module/cpyext/test/test_arraymodule.py b/pypy/module/cpyext/test/test_arraymodule.py
--- a/pypy/module/cpyext/test/test_arraymodule.py
+++ b/pypy/module/cpyext/test/test_arraymodule.py
@@ -25,27 +25,27 @@
 
     def test_index(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
+        arr = module.array('i', [1, 2, 3, 4])
         assert arr[3] == 4
         raises(IndexError, arr.__getitem__, 10)
         del arr[2]
-        assert arr.tolist() == [1,2,4]
+        assert arr.tolist() == [1, 2, 4]
         arr[2] = 99
-        assert arr.tolist() == [1,2,99]
+        assert arr.tolist() == [1, 2, 99]
 
     def test_slice_get(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
-        assert arr[:].tolist() == [1,2,3,4]
-        assert arr[1:].tolist() == [2,3,4]
-        assert arr[:2].tolist() == [1,2]
-        assert arr[1:3].tolist() == [2,3]
+        arr = module.array('i', [1, 2, 3, 4])
+        assert arr[:].tolist() == [1, 2, 3, 4]
+        assert arr[1:].tolist() == [2, 3, 4]
+        assert arr[:2].tolist() == [1, 2]
+        assert arr[1:3].tolist() == [2, 3]
 
     def test_slice_object(self):
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
-        assert arr[slice(1,3)].tolist() == [2,3]
-        arr[slice(1,3)] = module.array('i', [21, 22, 23])
+        arr = module.array('i', [1, 2, 3, 4])
+        assert arr[slice(1, 3)].tolist() == [2,3]
+        arr[slice(1, 3)] = module.array('i', [21, 22, 23])
         assert arr.tolist() == [1, 21, 22, 23, 4]
         del arr[slice(1, 3)]
         assert arr.tolist() == [1, 23, 4]
@@ -54,20 +54,15 @@
     def test_buffer(self):
         import sys
         module = self.import_module(name='array')
-        arr = module.array('i', [1,2,3,4])
+        arr = module.array('i', [1, 2, 3, 4])
         buf = memoryview(arr)
         exc = raises(TypeError, "buf[1] = 1")
         assert str(exc.value) == "cannot modify read-only memory"
         if sys.byteorder == 'big':
-            assert bytes(buf) == (b'\0\0\0\x01'
-                                b'\0\0\0\x02'
-                                b'\0\0\0\x03'
-                                b'\0\0\0\x04')
+            expected = b'\0\0\0\x01\0\0\0\x02\0\0\0\x03\0\0\0\x04'
         else:
-            assert bytes(buf) == (b'\x01\0\0\0'
-                                b'\x02\0\0\0'
-                                b'\x03\0\0\0'
-                                b'\x04\0\0\0')
+            expected = b'\x01\0\0\0\x02\0\0\0\x03\0\0\0\x04\0\0\0'
+        assert bytes(buf) == expected
 
     def test_releasebuffer(self):
         module = self.import_module(name='array')
diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -142,10 +142,9 @@
     BoolOption("verbose", "Print extra information", default=False,
                cmdline="--verbose"),
     StrOption("cc", "Specify compiler to use for compiling generated C", cmdline="--cc"),
-    StrOption("profopt", "Specify profile based optimization script",
-              cmdline="--profopt"),
-    BoolOption("noprofopt", "Don't use profile based optimization",
-               default=False, cmdline="--no-profopt", negation=False),
+    BoolOption("profopt", "Enable profile guided optimization. Defaults to enabling this for PyPy. For other training workloads, please specify them in profoptargs",
+              cmdline="--profopt", default=False),
+    StrOption("profoptargs", "Absolute path to the profile guided optimization training script + the necessary arguments of the script", cmdline="--profoptargs", default=None),
     BoolOption("instrument", "internal: turn instrumentation on",
                default=False, cmdline=None),
     BoolOption("countmallocs", "Count mallocs and frees", default=False,
diff --git a/rpython/jit/backend/arm/test/test_llop.py b/rpython/jit/backend/arm/test/test_llop.py
--- a/rpython/jit/backend/arm/test/test_llop.py
+++ b/rpython/jit/backend/arm/test/test_llop.py
@@ -5,5 +5,9 @@
 class TestLLOp(JitARMMixin, _TestLLOp):
     # for the individual tests see
     # ====> ../../../metainterp/test/test_llop.py
-    pass
 
+    # do NOT test the blackhole implementation of gc_store_indexed. It cannot
+    # work inside tests because llmodel.py:bh_gc_store_indexed_* receive a
+    # symbolic as the offset. It is not a problem because it is tested anyway
+    # by the same test in test_metainterp.py
+    TEST_BLACKHOLE = False
diff --git a/rpython/translator/c/genc.py b/rpython/translator/c/genc.py
--- a/rpython/translator/c/genc.py
+++ b/rpython/translator/c/genc.py
@@ -33,29 +33,6 @@
     return python
 
 
-class ProfOpt(object):
-    #XXX assuming gcc style flags for now
-    name = "profopt"
-
-    def __init__(self, compiler):
-        self.compiler = compiler
-
-    def first(self):
-        return self.build('-fprofile-generate')
-
-    def probe(self, exe, args):
-        # 'args' is a single string typically containing spaces
-        # and quotes, which represents several arguments.
-        self.compiler.platform.execute(exe, args)
-
-    def after(self):
-        return self.build('-fprofile-use')
-
-    def build(self, option):
-        eci = ExternalCompilationInfo(compile_extra=[option],
-                                      link_extra=[option])
-        return self.compiler._build(eci)
-
 class CCompilerDriver(object):
     def __init__(self, platform, cfiles, eci, outputfilename=None,
                  profbased=False):
@@ -65,7 +42,7 @@
         self.cfiles = cfiles
         self.eci = eci
         self.outputfilename = outputfilename
-        self.profbased = profbased
+        # self.profbased = profbased
 
     def _build(self, eci=ExternalCompilationInfo(), shared=False):
         outputfilename = self.outputfilename
@@ -79,22 +56,6 @@
                                      outputfilename=outputfilename,
                                      standalone=not shared)
 
-    def build(self, shared=False):
-        if self.profbased:
-            return self._do_profbased()
-        return self._build(shared=shared)
-
-    def _do_profbased(self):
-        ProfDriver, args = self.profbased
-        profdrv = ProfDriver(self)
-        dolog = getattr(log, profdrv.name)
-        dolog(args)
-        exename = profdrv.first()
-        dolog('Gathering profile data from: %s %s' % (
-            str(exename), args))
-        profdrv.probe(exename, args)
-        return profdrv.after()
-
 class CBuilder(object):
     c_source_filename = None
     _compiled = False
@@ -259,27 +220,6 @@
     _entrypoint_wrapper = None
     make_entrypoint_wrapper = True    # for tests
 
-    def getprofbased(self):
-        profbased = None
-        if self.config.translation.instrumentctl is not None:
-            profbased = self.config.translation.instrumentctl
-        else:
-            # xxx handling config.translation.profopt is a bit messy, because
-            # it could be an empty string (not to be confused with None) and
-            # because noprofopt can be used as an override.
-            profopt = self.config.translation.profopt
-            if profopt is not None and not self.config.translation.noprofopt:
-                profbased = (ProfOpt, profopt)
-        return profbased
-
-    def has_profopt(self):
-        profbased = self.getprofbased()
-        retval = (profbased and isinstance(profbased, tuple)
-                and profbased[0] is ProfOpt)
-        if retval and self.translator.platform.name == 'msvc':
-            raise ValueError('Cannot do profile based optimization on MSVC,'
-                    'it is not supported in free compiler version')
-        return retval
 
     def getentrypointptr(self):
         # XXX check that the entrypoint has the correct
@@ -391,6 +331,8 @@
         shared = self.config.translation.shared
 
         extra_opts = []
+        if self.config.translation.profopt:
+            extra_opts += ["profopt"]
         if self.config.translation.make_jobs != 1:
             extra_opts += ['-j', str(self.config.translation.make_jobs)]
         if self.config.translation.lldebug:
@@ -418,13 +360,12 @@
             headers_to_precompile=headers_to_precompile,
             no_precompile_cfiles = module_files,
             shared=self.config.translation.shared,
+            profopt = self.config.translation.profopt,
             config=self.config)
 
-        if self.has_profopt():
-            profopt = self.config.translation.profopt
-            mk.definition('ABS_TARGET', str(targetdir.join('$(TARGET)')))
-            mk.definition('DEFAULT_TARGET', 'profopt')
-            mk.definition('PROFOPT', profopt)
+        if exe_name is None:
+            short =  targetdir.basename
+            exe_name = targetdir.join(short)
 
         rules = [
             ('debug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT" debug_target'),
@@ -433,15 +374,28 @@
             ('llsafer', '', '$(MAKE) CFLAGS="-O2 -DRPY_LL_ASSERT" $(DEFAULT_TARGET)'),
             ('lldebug', '', '$(MAKE) CFLAGS="$(DEBUGFLAGS) -DRPY_ASSERT -DRPY_LL_ASSERT" debug_target'),
             ('profile', '', '$(MAKE) CFLAGS="-g -O1 -pg $(CFLAGS) -fno-omit-frame-pointer" LDFLAGS="-pg $(LDFLAGS)" $(DEFAULT_TARGET)'),
-            ]
-        if self.has_profopt():
+        ]
+
+        # added a new target for profopt, because it requires -lgcov to compile successfully when -shared is used as an argument
+        # Also made a difference between translating with shared or not, because this affects profopt's target
+
+        if self.config.translation.profopt:
+            if self.config.translation.profoptargs is None:
+                raise Exception("No profoptargs specified, neither in the command line, nor in the target. If the target is not PyPy, please specify profoptargs")
+            if self.config.translation.shared:
+                mk.rule('$(PROFOPT_TARGET)', '$(TARGET) main.o',
+                         '$(CC_LINK) $(LDFLAGS_LINK) main.o -L. -l$(SHARED_IMPORT_LIB) -o $@ $(RPATH_FLAGS) -lgcov')
+            else:
+                mk.definition('PROFOPT_TARGET', '$(TARGET)')
+
             rules.append(
                 ('profopt', '', [
-                '$(MAKENOPROF)',
-                '$(MAKE) CFLAGS="-fprofile-generate $(CFLAGS)" LDFLAGS="-fprofile-generate $(LDFLAGS)" $(TARGET)',
-                'cd $(RPYDIR)/translator/goal && $(ABS_TARGET) $(PROFOPT)',
-                '$(MAKE) clean_noprof',
-                '$(MAKE) CFLAGS="-fprofile-use $(CFLAGS)" LDFLAGS="-fprofile-use $(LDFLAGS)" $(TARGET)']))
+                    '$(MAKE) CFLAGS="-fprofile-generate -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-generate $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+                    '%s %s ' % (exe_name, self.config.translation.profoptargs),
+                    '$(MAKE) clean_noprof',
+                    '$(MAKE) CFLAGS="-fprofile-use -fprofile-correction -fPIC $(CFLAGS) -fno-lto"  LDFLAGS="-fprofile-use $(LDFLAGS) -fno-lto" $(PROFOPT_TARGET)',
+                ]))
+
         for rule in rules:
             mk.rule(*rule)
 
diff --git a/rpython/translator/c/test/test_standalone.py b/rpython/translator/c/test/test_standalone.py
--- a/rpython/translator/c/test/test_standalone.py
+++ b/rpython/translator/c/test/test_standalone.py
@@ -241,20 +241,22 @@
             os.write(1, str(tot))
             return 0
         from rpython.translator.interactive import Translation
-        # XXX this is mostly a "does not crash option"
-        t = Translation(entry_point, backend='c', profopt="100")
-        # no counters
+        t = Translation(entry_point, backend='c', profopt=True, profoptargs="10", shared=True)
         t.backendopt()
         exe = t.compile()
-        out = py.process.cmdexec("%s 500" % exe)
-        assert int(out) == 500*501/2
-        t = Translation(entry_point, backend='c', profopt="100",
-                        noprofopt=True)
-        # no counters
+        assert (os.path.isfile("%s" % exe))
+
+        t = Translation(entry_point, backend='c', profopt=True, profoptargs="10", shared=False)
         t.backendopt()
         exe = t.compile()
-        out = py.process.cmdexec("%s 500" % exe)
-        assert int(out) == 500*501/2
+        assert (os.path.isfile("%s" % exe))
+
+        import rpython.translator.goal.targetrpystonedalone as rpy
+        t = Translation(rpy.entry_point, backend='c', profopt=True, profoptargs='1000', shared=False)
+        t.backendopt()
+        exe = t.compile()
+        assert (os.path.isfile("%s" % exe))
+
 
     if hasattr(os, 'setpgrp'):
         def test_os_setpgrp(self):
@@ -279,13 +281,12 @@
             return 0
         from rpython.translator.interactive import Translation
         # XXX this is mostly a "does not crash option"
-        t = Translation(entry_point, backend='c', profopt="")
+        t = Translation(entry_point, backend='c', profopt=True, profoptargs='10', shared=True)
         # no counters
         t.backendopt()
         exe = t.compile()
         #py.process.cmdexec(exe)
-        t = Translation(entry_point, backend='c', profopt="",
-                        noprofopt=True)
+        t = Translation(entry_point, backend='c', profopt=True, profoptargs='10', shared=True)
         # no counters
         t.backendopt()
         exe = t.compile()
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -103,7 +103,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         cfiles = self._all_cfiles(cfiles, eci)
 
         if path is None:
@@ -189,6 +189,10 @@
             ('LINKFILES', eci.link_files),
             ('RPATH_FLAGS', self.get_rpath_flags(rel_libdirs)),
             ]
+
+        if profopt==True and shared==True:
+            definitions.append(('PROFOPT_TARGET', exe_name.basename))
+
         for args in definitions:
             m.definition(*args)
 

From pypy.commits at gmail.com  Thu May 25 21:11:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 25 May 2017 18:11:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: __wbuffer__ and __rbuffer__ aren't used
 in pypy3
Message-ID: <592780de.f7abdf0a.cd41e.0def@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91413:10d7da5ff46a
Date: 2017-05-26 02:11 +0100
http://bitbucket.org/pypy/pypy/changeset/10d7da5ff46a/

Log:	__wbuffer__ and __rbuffer__ aren't used in pypy3

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -218,12 +218,7 @@
         return self.__buffer_w(space, flags).buffer_w(space, flags)
 
     def __buffer_w(self, space, flags):
-        if flags & space.BUF_WRITABLE:
-            w_impl = space.lookup(self, '__wbuffer__')
-        else:
-            w_impl = space.lookup(self, '__rbuffer__')
-        if w_impl is None:
-            w_impl = space.lookup(self, '__buffer__')
+        w_impl = space.lookup(self, '__buffer__')
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                                    space.newint(flags))

From pypy.commits at gmail.com  Thu May 25 21:36:36 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 25 May 2017 18:36:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: update test with current pypy error
 messages
Message-ID: <592786a4.10a4df0a.e1b04.4a41@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91414:b10bce24eb85
Date: 2017-05-26 02:36 +0100
http://bitbucket.org/pypy/pypy/changeset/b10bce24eb85/

Log:	update test with current pypy error messages

diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py
--- a/lib-python/3/test/test_socket.py
+++ b/lib-python/3/test/test_socket.py
@@ -722,12 +722,12 @@
             s.sendto('\u2620', sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'str'", # cpython
-                       "'str' does not support the buffer interface"]) # pypy
+                       "a bytes-like object is required, not str"]) # pypy
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'complex'",
-                       "'complex' does not support the buffer interface"])
+                       "a bytes-like object is required, not complex"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', None)
         self.assertIn('NoneType', str(cm.exception))
@@ -736,12 +736,12 @@
             s.sendto('\u2620', 0, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'str'",
-                       "'str' does not support the buffer interface"])
+                       "a bytes-like object is required, not str"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'complex'",
-                       "'complex' does not support the buffer interface"])
+                       "a bytes-like object is required, not complex"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 0, None)
         self.assertIn('NoneType', str(cm.exception))

From pypy.commits at gmail.com  Fri May 26 07:16:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 26 May 2017 04:16:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: No clue how this missing @specialize
 was not noticed earlier.
Message-ID: <59280e76.07a8df0a.23ef.5247@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91415:f7c9422b741c
Date: 2017-05-26 13:15 +0200
http://bitbucket.org/pypy/pypy/changeset/f7c9422b741c/

Log:	No clue how this missing @specialize was not noticed earlier.

diff --git a/rpython/rlib/objectmodel.py b/rpython/rlib/objectmodel.py
--- a/rpython/rlib/objectmodel.py
+++ b/rpython/rlib/objectmodel.py
@@ -514,6 +514,7 @@
 
 # ----------
 
+ at specialize.ll()
 def _hash_string(s):
     """The default algorithm behind compute_hash() for a string or a unicode.
     This is a modified Fowler-Noll-Vo (FNV) hash.  According to Wikipedia,

From pypy.commits at gmail.com  Fri May 26 11:02:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 26 May 2017 08:02:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: do not go through
 descroperation.binop_impl again, call descr_add directly
Message-ID: <59284369.6c8edf0a.2655e.9c8f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91418:7328ebef4d73
Date: 2017-05-26 16:38 +0300
http://bitbucket.org/pypy/pypy/changeset/7328ebef4d73/

Log:	do not go through descroperation.binop_impl again, call descr_add
	directly

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -648,7 +648,7 @@
         if space.isinstance_w(w_other, space.w_unicode):
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
-            return space.add(self_as_unicode, w_other)
+            return self_as_unicode.descr_add(space, w_other)
         elif space.isinstance_w(w_other, space.w_bytearray):
             # XXX: eliminate double-copy
             from .bytearrayobject import W_BytearrayObject, _make_data
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -846,3 +846,7 @@
                 return 42
         x = Foo()
         assert "hello" + x == 42
+
+    def test_add(self):
+        assert 'abc' + 'abc' == 'abcabc'
+        assert isinstance('abc' + u'\u03a3', unicode)

From pypy.commits at gmail.com  Fri May 26 11:02:03 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 26 May 2017 08:02:03 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: close exploratory branch
Message-ID: <5928436b.ddb1df0a.13808.881a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91419:3071469e3277
Date: 2017-05-26 18:00 +0300
http://bitbucket.org/pypy/pypy/changeset/3071469e3277/

Log:	close exploratory branch


From pypy.commits at gmail.com  Fri May 26 11:01:59 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 26 May 2017 08:01:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: more places to convert hash value -1 to
 -2, see comments to issue #2346
Message-ID: <59284367.cd711c0a.a2d2e.9c1c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91417:fb4031e88779
Date: 2017-05-26 00:36 +0300
http://bitbucket.org/pypy/pypy/changeset/fb4031e88779/

Log:	more places to convert hash value -1 to -2, see comments to issue
	#2346

diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -112,7 +112,9 @@
     descr_ge = _make_descr__cmp('ge')
 
     def descr_hash(self, space):
-        return space.newint(compute_hash(self.buf.as_str()))
+        x = compute_hash(self.buf.as_str())
+        x -= (x == -1) # convert -1 to -2 without creating a bridge
+        return space.newint(x)
 
     def descr_mul(self, space, w_times):
         # xxx not the most efficient implementation
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -592,6 +592,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1) # convert -1 to -2 without creating a bridge
         return space.newint(x)
 
     def descr_format(self, space, __args__):
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -239,6 +239,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1) # convert -1 to -2 without creating a bridge
         return space.newint(x)
 
     def descr_eq(self, space, w_other):

From pypy.commits at gmail.com  Fri May 26 11:01:57 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 26 May 2017 08:01:57 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-unhashable: more places to ensure cpyext
 hashing an empty string-like does not return -1
Message-ID: <59284365.c299df0a.282ac.bc29@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-unhashable
Changeset: r91416:46815e12a7df
Date: 2017-05-22 23:04 +0300
http://bitbucket.org/pypy/pypy/changeset/46815e12a7df/

Log:	more places to ensure cpyext hashing an empty string-like does not
	return -1

diff --git a/pypy/objspace/std/bufferobject.py b/pypy/objspace/std/bufferobject.py
--- a/pypy/objspace/std/bufferobject.py
+++ b/pypy/objspace/std/bufferobject.py
@@ -112,7 +112,9 @@
     descr_ge = _make_descr__cmp('ge')
 
     def descr_hash(self, space):
-        return space.newint(compute_hash(self.buf.as_str()))
+        x = compute_hash(self.buf.as_str())
+        x -= (x == -1)
+        return space.newint(x)
 
     def descr_mul(self, space, w_times):
         # xxx not the most efficient implementation
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -236,6 +236,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1)
         return space.newint(x)
 
     def descr_eq(self, space, w_other):

From pypy.commits at gmail.com  Sat May 27 06:06:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 May 2017 03:06:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix a rare stacklet issue that was
 there since a while with shadowstack.
Message-ID: <59294fb2.493e1c0a.68e4f.bec5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91420:530b46ef1b95
Date: 2017-05-27 12:06 +0200
http://bitbucket.org/pypy/pypy/changeset/530b46ef1b95/

Log:	Fix a rare stacklet issue that was there since a while with
	shadowstack. This is basically the same as b2569cf1bd55 was for
	threads.

diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -925,6 +925,10 @@
     def gct_gc_adr_of_root_stack_top(self, hop):
         self._gc_adr_of_gcdata_attr(hop, 'root_stack_top')
 
+    def gct_gc_modified_shadowstack(self, hop):
+        # for stacklet
+        hop.genop("direct_call", [self.root_walker.gc_modified_shadowstack_ptr])
+
     def gct_gc_detach_callback_pieces(self, hop):
         op = hop.spaceop
         assert len(op.args) == 0
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -245,6 +245,13 @@
         from rpython.rlib import _stacklet_shadowstack
         _stacklet_shadowstack.complete_destrptr(gctransformer)
 
+        gcdata = self.gcdata
+        def gc_modified_shadowstack():
+            gcdata.can_look_at_partial_stack = False
+
+        self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
+                                                 [], annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
diff --git a/rpython/rlib/_stacklet_shadowstack.py b/rpython/rlib/_stacklet_shadowstack.py
--- a/rpython/rlib/_stacklet_shadowstack.py
+++ b/rpython/rlib/_stacklet_shadowstack.py
@@ -77,6 +77,7 @@
     llmemory.raw_memcopy(sscopy + SIZEADDR, base, length_bytes)
     llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] = (
         base + length_bytes)
+    llop.gc_modified_shadowstack(lltype.Void)
     llmemory.raw_free(sscopy)
 
 def alloc_stacklet():
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -515,6 +515,7 @@
     'gc_adr_of_root_stack_base': LLOp(),
     'gc_adr_of_root_stack_top': LLOp(),
     # returns the address of gcdata.root_stack_base/top (for shadowstack only)
+    'gc_modified_shadowstack': LLOp(),
 
     # for asmgcroot support to get the address of various static structures
     # see translator/c/src/mem.h for the valid indices

From pypy.commits at gmail.com  Sat May 27 06:16:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 May 2017 03:16:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix for PR #540 on non-Linux platforms
Message-ID: <59295217.8f9d1c0a.1464b.369f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91421:41f544557a61
Date: 2017-05-27 12:16 +0200
http://bitbucket.org/pypy/pypy/changeset/41f544557a61/

Log:	Fix for PR #540 on non-Linux platforms

diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -102,7 +102,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         raise NotImplementedError("Pure abstract baseclass")
 
     def __repr__(self):
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -96,7 +96,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         # ensure frameworks are passed in the Makefile
         fs = self._frameworks(eci.frameworks)
         if len(fs) > 0:
@@ -106,7 +106,7 @@
                                 shared=shared,
                                 headers_to_precompile=headers_to_precompile,
                                 no_precompile_cfiles = no_precompile_cfiles,
-                                config=config)
+                                profopt=profopt, config=config)
         return mk
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -299,7 +299,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         cfiles = self._all_cfiles(cfiles, eci)
 
         if path is None:

From pypy.commits at gmail.com  Sat May 27 07:52:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 27 May 2017 04:52:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a marker into the shadowstack from
 the JIT. Might improve performance in some cases
Message-ID: <5929689b.899adf0a.997e.e155@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91422:a88434fd16b8
Date: 2017-05-27 12:52 +0100
http://bitbucket.org/pypy/pypy/changeset/a88434fd16b8/

Log:	Add a marker into the shadowstack from the JIT. Might improve
	performance in some cases

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1052,17 +1052,19 @@
 
     def _call_header_shadowstack(self, gcrootmap):
         rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap)
-        self.mc.MOV_mr((ebx.value, 0), ebp.value)      # MOV [ebx], ebp
-        self.mc.ADD_ri(ebx.value, WORD)
+        # the '1' is to benefit from the shadowstack 'is_minor' optimization
+        self.mc.MOV_mi((ebx.value, 0), 1)               # MOV [ebx], 1
+        self.mc.MOV_mr((ebx.value, WORD), ebp.value)    # MOV [ebx + WORD], ebp
+        self.mc.ADD_ri(ebx.value, WORD * 2)
         self.mc.MOV(heap(rst), ebx)                   # MOV [rootstacktop], ebx
 
     def _call_footer_shadowstack(self, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
         if rx86.fits_in_32bits(rst):
-            self.mc.SUB_ji8(rst, WORD)       # SUB [rootstacktop], WORD
+            self.mc.SUB_ji8(rst, WORD * 2)       # SUB [rootstacktop], WORD * 2
         else:
             self.mc.MOV_ri(ebx.value, rst)           # MOV ebx, rootstacktop
-            self.mc.SUB_mi8((ebx.value, 0), WORD)  # SUB [ebx], WORD
+            self.mc.SUB_mi8((ebx.value, 0), WORD * 2)  # SUB [ebx], WORD * 2
 
     def redirect_call_assembler(self, oldlooptoken, newlooptoken):
         # some minimal sanity checking

From pypy.commits at gmail.com  Sat May 27 22:54:29 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 May 2017 19:54:29 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Rename ArrayData to ArrayBuffer
Message-ID: <592a3be5.9d97df0a.6df51.212d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91423:c1a9982a0266
Date: 2017-05-28 03:54 +0100
http://bitbucket.org/pypy/pypy/changeset/c1a9982a0266/

Log:	Rename ArrayData to ArrayBuffer

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -258,7 +258,7 @@
             lltype.free(oldbuffer, flavor='raw')
 
     def buffer_w(self, space, flags):
-        return ArrayView(ArrayData(self), self.typecode, self.itemsize, False)
+        return ArrayView(ArrayBuffer(self), self.typecode, self.itemsize, False)
 
     def descr_append(self, space, w_x):
         """ append(x)
@@ -848,7 +848,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayData(RawBuffer):
+class ArrayBuffer(RawBuffer):
     _immutable_ = True
     readonly = False
     def __init__(self, w_array):

From pypy.commits at gmail.com  Sat May 27 23:00:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 27 May 2017 20:00:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Reduce diff with py3.5
Message-ID: <592a3d39.2196df0a.cd98.37a0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91424:0e27a73c87f6
Date: 2017-05-28 03:50 +0100
http://bitbucket.org/pypy/pypy/changeset/0e27a73c87f6/

Log:	Reduce diff with py3.5

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -799,51 +799,39 @@
 class ArrayBuffer(RawBuffer):
     _immutable_ = True
 
-    def __init__(self, array, readonly):
-        self.array = array
+    def __init__(self, w_array, readonly):
+        self.w_array = w_array
         self.readonly = readonly
 
     def getlength(self):
-        return self.array.len * self.array.itemsize
-
-    def getformat(self):
-        return self.array.typecode
-
-    def getitemsize(self):
-        return self.array.itemsize
-
-    def getndim(self):
-        return 1
-
-    def getstrides(self):
-        return [self.getitemsize()]
+        return self.w_array.len * self.w_array.itemsize
 
     def getitem(self, index):
-        array = self.array
-        data = array._charbuf_start()
+        w_array = self.w_array
+        data = w_array._charbuf_start()
         char = data[index]
-        array._charbuf_stop()
+        w_array._charbuf_stop()
         return char
 
     def setitem(self, index, char):
-        array = self.array
-        data = array._charbuf_start()
+        w_array = self.w_array
+        data = w_array._charbuf_start()
         data[index] = char
-        array._charbuf_stop()
+        w_array._charbuf_stop()
 
     def getslice(self, start, stop, step, size):
         if size == 0:
             return ''
         if step == 1:
-            data = self.array._charbuf_start()
+            data = self.w_array._charbuf_start()
             try:
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
-                self.array._charbuf_stop()
+                self.w_array._charbuf_stop()
         return RawBuffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
-        return self.array._charbuf_start()
+        return self.w_array._charbuf_start()
 
 
 unpack_driver = jit.JitDriver(name='unpack_array',

From pypy.commits at gmail.com  Sun May 28 03:52:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 00:52:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: Argh,
 fix test that was not testing anything
Message-ID: <592a81ab.4297df0a.9a729.89e7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91426:672b8a624c32
Date: 2017-05-28 09:51 +0200
http://bitbucket.org/pypy/pypy/changeset/672b8a624c32/

Log:	Argh, fix test that was not testing anything

diff --git a/rpython/rlib/test/test_rfloat.py b/rpython/rlib/test/test_rfloat.py
--- a/rpython/rlib/test/test_rfloat.py
+++ b/rpython/rlib/test/test_rfloat.py
@@ -44,35 +44,35 @@
     almost_equal(round_double(-0.25, 1), -0.3)
     almost_equal(round_double(-0.75, 1), -0.8)
 
-    round_double(-6.5, 0) == -7.0
-    round_double(-5.5, 0) == -6.0
-    round_double(-1.5, 0) == -2.0
-    round_double(-0.5, 0) == -1.0
-    round_double(0.5, 0) == 1.0
-    round_double(1.5, 0) == 2.0
-    round_double(2.5, 0) == 3.0
-    round_double(3.5, 0) == 4.0
-    round_double(4.5, 0) == 5.0
-    round_double(5.5, 0) == 6.0
-    round_double(6.5, 0) == 7.0
+    assert round_double(-6.5, 0) == -7.0
+    assert round_double(-5.5, 0) == -6.0
+    assert round_double(-1.5, 0) == -2.0
+    assert round_double(-0.5, 0) == -1.0
+    assert round_double(0.5, 0) == 1.0
+    assert round_double(1.5, 0) == 2.0
+    assert round_double(2.5, 0) == 3.0
+    assert round_double(3.5, 0) == 4.0
+    assert round_double(4.5, 0) == 5.0
+    assert round_double(5.5, 0) == 6.0
+    assert round_double(6.5, 0) == 7.0
 
-    round_double(-25.0, -1) == -30.0
-    round_double(-15.0, -1) == -20.0
-    round_double(-5.0, -1) == -10.0
-    round_double(5.0, -1) == 10.0
-    round_double(15.0, -1) == 20.0
-    round_double(25.0, -1) == 30.0
-    round_double(35.0, -1) == 40.0
-    round_double(45.0, -1) == 50.0
-    round_double(55.0, -1) == 60.0
-    round_double(65.0, -1) == 70.0
-    round_double(75.0, -1) == 80.0
-    round_double(85.0, -1) == 90.0
-    round_double(95.0, -1) == 100.0
-    round_double(12325.0, -1) == 12330.0
+    assert round_double(-25.0, -1) == -30.0
+    assert round_double(-15.0, -1) == -20.0
+    assert round_double(-5.0, -1) == -10.0
+    assert round_double(5.0, -1) == 10.0
+    assert round_double(15.0, -1) == 20.0
+    assert round_double(25.0, -1) == 30.0
+    assert round_double(35.0, -1) == 40.0
+    assert round_double(45.0, -1) == 50.0
+    assert round_double(55.0, -1) == 60.0
+    assert round_double(65.0, -1) == 70.0
+    assert round_double(75.0, -1) == 80.0
+    assert round_double(85.0, -1) == 90.0
+    assert round_double(95.0, -1) == 100.0
+    assert round_double(12325.0, -1) == 12330.0
 
-    round_double(350.0, -2) == 400.0
-    round_double(450.0, -2) == 500.0
+    assert round_double(350.0, -2) == 400.0
+    assert round_double(450.0, -2) == 500.0
 
     almost_equal(round_double(0.5e21, -21), 1e21)
     almost_equal(round_double(1.5e21, -21), 2e21)

From pypy.commits at gmail.com  Sun May 28 04:18:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 01:18:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: Performance tweaks to round(x,
 n) for the case n == 0
Message-ID: <592a87e9.3098df0a.f3c63.855a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91427:3b55e802a373
Date: 2017-05-28 10:18 +0200
http://bitbucket.org/pypy/pypy/changeset/3b55e802a373/

Log:	Performance tweaks to round(x, n) for the case n == 0

diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.runicode import UNICHR
-from rpython.rlib.rfloat import isnan, isinf, round_double
+from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away
 from rpython.rlib import rfloat
 import __builtin__
 
@@ -134,23 +134,26 @@
     ndigits = space.getindex_w(w_ndigits, None)
 
     # nans, infinities and zeros round to themselves
-    if number == 0 or isinf(number) or isnan(number):
-        return space.newfloat(number)
-
-    # Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
-    # always rounds to itself.  For ndigits < NDIGITS_MIN, x always
-    # rounds to +-0.0.
-    if ndigits > NDIGITS_MAX:
-        return space.newfloat(number)
-    elif ndigits < NDIGITS_MIN:
-        # return 0.0, but with sign of x
-        return space.newfloat(0.0 * number)
-
-    # finite x, and ndigits is not unreasonably large
-    z = round_double(number, ndigits)
-    if isinf(z):
-        raise oefmt(space.w_OverflowError,
-                    "rounded value too large to represent")
+    if not isfinite(number):
+        z = number
+    elif ndigits == 0:    # common case
+        z = round_away(number)
+        # no need to check for an infinite 'z' here
+    else:
+        # Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
+        # always rounds to itself.  For ndigits < NDIGITS_MIN, x always
+        # rounds to +-0.0.
+        if ndigits > NDIGITS_MAX:
+            z = number
+        elif ndigits < NDIGITS_MIN:
+            # return 0.0, but with sign of x
+            z = 0.0 * number
+        else:
+            # finite x, and ndigits is not unreasonably large
+            z = round_double(number, ndigits)
+            if isinf(z):
+                raise oefmt(space.w_OverflowError,
+                            "rounded value too large to represent")
     return space.newfloat(z)
 
 # ____________________________________________________________
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -625,6 +625,9 @@
         assert round(5e15) == 5e15
         assert round(-(5e15-1)) == -(5e15-1)
         assert round(-5e15) == -5e15
+        assert round(5e15/2) == 5e15/2
+        assert round((5e15+1)/2) == 5e15/2+1
+        assert round((5e15-1)/2) == 5e15/2
         #
         inf = 1e200 * 1e200
         assert round(inf) == inf
@@ -636,6 +639,12 @@
         #
         assert round(562949953421312.5, 1) == 562949953421312.5
         assert round(56294995342131.5, 3) == 56294995342131.5
+        #
+        for i in range(-10, 10):
+            expected = i if i < 0 else i + 1
+            assert round(i + 0.5) == round(i + 0.5, 0) == expected
+            x = i * 10 + 5
+            assert round(x, -1) == round(float(x), -1) == expected * 10
 
     def test_vars_obscure_case(self):
         class C_get_vars(object):
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -96,7 +96,20 @@
     """Round a float half away from zero.
 
     Specify half_even=True to round half even instead.
+    The argument 'value' must be a finite number.  This
+    function may return an infinite number in case of
+    overflow (only if ndigits is a very negative integer).
     """
+    if ndigits == 0:
+        # fast path for this common case
+        if half_even:
+            return round_half_even(value)
+        else:
+            return round_away(value)
+
+    if value == 0.0:
+        return 0.0
+
     # The basic idea is very simple: convert and round the double to
     # a decimal string using _Py_dg_dtoa, then convert that decimal
     # string back to a double with _Py_dg_strtod.  There's one minor
@@ -217,11 +230,34 @@
 
 def round_away(x):
     # round() from libm, which is not available on all platforms!
+    # This version rounds away from zero.
     absx = abs(x)
-    if absx - math.floor(absx) >= .5:
-        r = math.ceil(absx)
+    r = math.floor(absx + 0.5)
+    if r - absx < 1.0:
+        return copysign(r, x)
     else:
-        r = math.floor(absx)
+        # 'absx' is just in the wrong range: its exponent is precisely
+        # the one for which all integers are representable but not any
+        # half-integer.  It means that 'absx + 0.5' computes equal to
+        # 'absx + 1.0', which is not equal to 'absx'.  So 'r - absx'
+        # computes equal to 1.0.  In this situation, we can't return
+        # 'r' because 'absx' was already an integer but 'r' is the next
+        # integer!  But just returning the original 'x' is fine.
+        return x
+
+def round_half_even(x):
+    absx = abs(x)
+    r = math.floor(absx + 0.5)
+    frac = r - absx
+    if frac >= 0.5:
+        # two rare cases: either 'absx' is precisely half-way between
+        # two integers (frac == 0.5); or we're in the same situation as
+        # described in round_away above (frac == 1.0).
+        if frac >= 1.0:
+            return x
+        # absx == n + 0.5  for a non-negative integer 'n'
+        # absx * 0.5 == n//2 + 0.25 or 0.75, which we round to nearest
+        r = math.floor(absx * 0.5 + 0.5) * 2.0
     return copysign(r, x)
 
 @not_rpython
diff --git a/rpython/rlib/test/test_rfloat.py b/rpython/rlib/test/test_rfloat.py
--- a/rpython/rlib/test/test_rfloat.py
+++ b/rpython/rlib/test/test_rfloat.py
@@ -28,7 +28,7 @@
 
 def test_round_double():
     def almost_equal(x, y):
-        assert round(abs(x-y), 7) == 0
+        assert abs(x-y) < 1e-7
 
     almost_equal(round_double(0.125, 2), 0.13)
     almost_equal(round_double(0.375, 2), 0.38)
@@ -85,6 +85,13 @@
     almost_equal(round_double(0.5e22, -22), 1e22)
     almost_equal(round_double(1.5e22, -22), 2e22)
 
+    exact_integral = 5e15 + 1
+    assert round_double(exact_integral, 0) == exact_integral
+    assert round_double(exact_integral/2.0, 0) == 5e15/2.0 + 1.0
+    exact_integral = 5e15 - 1
+    assert round_double(exact_integral, 0) == exact_integral
+    assert round_double(exact_integral/2.0, 0) == 5e15/2.0
+
 def test_round_half_even():
     from rpython.rlib import rfloat
     func = rfloat.round_double
@@ -92,6 +99,15 @@
     assert func(2.5, 0, False) == 3.0
     # 3.x behavior
     assert func(2.5, 0, True) == 2.0
+    for i in range(-10, 10):
+        assert func(i + 0.5, 0, True) == i + (i & 1)
+        assert func(i * 10 + 5, -1, True) == (i + (i & 1)) * 10
+    exact_integral = 5e15 + 1
+    assert round_double(exact_integral, 0, True) == exact_integral
+    assert round_double(exact_integral/2.0, 0, True) == 5e15/2.0
+    exact_integral = 5e15 - 1
+    assert round_double(exact_integral, 0, True) == exact_integral
+    assert round_double(exact_integral/2.0, 0, True) == 5e15/2.0
 
 def test_float_as_rbigint_ratio():
     for f, ratio in [

From pypy.commits at gmail.com  Sun May 28 10:35:05 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 28 May 2017 07:35:05 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branches
Message-ID: <592ae019.11d4500a.4507c.ddad@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91428:cf5a42dd00cb
Date: 2017-05-28 17:33 +0300
http://bitbucket.org/pypy/pypy/changeset/cf5a42dd00cb/

Log:	document merged branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -68,3 +68,10 @@
 or gc_store_indexed whenever possible. Moreover, enable the existing
 struct.unpack fast path to all the existing buffer types, whereas previously
 it was enabled only for strings
+
+.. branch: Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235
+
+.. branch: PyPy_profopt_enabled
+
+Add profile-based optimization option ``profopt``, and specify training data
+via ``profoptpath``

From pypy.commits at gmail.com  Sun May 28 10:43:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 07:43:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <592ae204.4186df0a.ec236.59ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91429:659988b39888
Date: 2017-05-28 16:42 +0200
http://bitbucket.org/pypy/pypy/changeset/659988b39888/

Log:	Fix test

diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -463,6 +463,21 @@
     def get_root_stack_top_addr(self):
         return rffi.cast(lltype.Signed, self.stack_addr)
 
+    def getlength(self):
+        top = self.stack_addr[0]
+        base = rffi.cast(lltype.Signed, self.stack)
+        n = (top - base) // WORD
+        assert 0 <= n < 10
+        return n
+
+    def curtop(self):
+        n = self.getlength()
+        return self.stack[n - 1]
+
+    def settop(self, newvalue):
+        n = self.getlength()
+        self.stack[n - 1] = newvalue
+
 class WriteBarrierDescr(AbstractDescr):
     jit_wb_cards_set = 0
     jit_wb_if_flag_singlebyte = 1
@@ -645,7 +660,7 @@
         frames = []
 
         def check(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert len(frame.jf_frame) == self.cpu.JITFRAME_FIXED_SIZE + 4
             # we "collect"
@@ -665,14 +680,14 @@
                 assert gcmap == [22, 23, 24]
             for item, s in zip(gcmap, new_items):
                 new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s)
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame)
-            cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame)
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == rffi.cast(lltype.Signed, frame)
+            cpu.gc_ll_descr.gcrootmap.settop(rffi.cast(lltype.Signed, new_frame))
             print '"Collecting" moved the frame from %d to %d' % (
-                i, cpu.gc_ll_descr.gcrootmap.stack[0])
+                i, cpu.gc_ll_descr.gcrootmap.curtop())
             frames.append(new_frame)
 
         def check2(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert frame == frames[1]
             assert frame != frames[0]

From pypy.commits at gmail.com  Sun May 28 10:44:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 07:44:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test
Message-ID: <592ae24c.89341c0a.6d70d.1470@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91430:0d24dcc53f09
Date: 2017-05-28 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/0d24dcc53f09/

Log:	Fix test

diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -862,6 +862,9 @@
     def op_gc_adr_of_root_stack_top(self):
         raise NotImplementedError
 
+    def op_gc_modified_shadowstack(self):
+        raise NotImplementedError
+
     def op_gc_call_rtti_destructor(self, rtti, addr):
         if hasattr(rtti._obj, 'destructor_funcptr'):
             d = rtti._obj.destructor_funcptr

From pypy.commits at gmail.com  Sun May 28 10:48:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 07:48:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: Skip the test about byte-swapped
 structures, for now
Message-ID: <592ae32c.0486df0a.e6705.2849@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91431:844e78f165f3
Date: 2017-05-28 16:47 +0200
http://bitbucket.org/pypy/pypy/changeset/844e78f165f3/

Log:	Skip the test about byte-swapped structures, for now

diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,7 +37,10 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            o = typ()
+            try:
+                o = typ()
+            except NotImplementedError as e:
+                self.skipTest(str(e))    # for PyPy
             o.value = 4
             self.assertEqual(o.value, 4)
 

From pypy.commits at gmail.com  Sun May 28 11:45:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 08:45:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix for 0e27a73c87f6
Message-ID: <592af09f.61a9df0a.e62d2.5d02@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91432:1e656d2369ef
Date: 2017-05-28 17:37 +0200
http://bitbucket.org/pypy/pypy/changeset/1e656d2369ef/

Log:	Fix for 0e27a73c87f6

diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -56,9 +56,9 @@
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(buf.value))
         py_buf.c_b_size = buf.getlength()
     elif isinstance(buf, ArrayBuffer):
-        w_base = buf.array
+        w_base = buf.w_array
         py_buf.c_b_base = make_ref(space, w_base)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start())
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.w_array._charbuf_start())
         py_buf.c_b_size = buf.getlength()
     else:
         raise oefmt(space.w_NotImplementedError, "buffer flavor not supported")

From pypy.commits at gmail.com  Sun May 28 12:41:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 28 May 2017 09:41:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: Make sure sys.prefix is always defined,
 even if it contains the
Message-ID: <592afdc2.acb6df0a.6dca4.924a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91433:ac573ca24687
Date: 2017-05-28 18:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ac573ca24687/

Log:	Make sure sys.prefix is always defined, even if it contains the
	translation-time value. Fix obscure issues where the PyPy is not
	usable even on the same machine. It will still emit the warning
	lines but at least work.

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -798,8 +798,8 @@
                               '"license" for more information.')
 
 STDLIB_WARNING = """\
-debug: WARNING: Library path not found, using compiled-in sys.path.
-debug: WARNING: 'sys.prefix' will not be set.
+debug: WARNING: Library path not found, using compiled-in sys.path, with
+debug: WARNING: sys.prefix = %r
 debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
 debug: WARNING: It is ok to create a symlink to it from somewhere else."""
 
@@ -818,12 +818,8 @@
     executable = sys.pypy_find_executable(executable)
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
-        for lib_path in sys.path:
-            stdlib_path = sys.pypy_find_stdlib(lib_path)
-            if stdlib_path is not None:
-                break
-    if stdlib_path is None:
-        print >> sys.stderr, STDLIB_WARNING
+        print >> sys.stderr, STDLIB_WARNING % (
+            getattr(sys, 'prefix', '<missing>'),)
     else:
         sys.path[:] = stdlib_path
     # from this point on, we are free to use all the unicode stuff we want,
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -39,8 +39,8 @@
         'stderr'                : 'state.getio(space).w_stderr',
         '__stderr__'            : 'state.getio(space).w_stderr',
         'pypy_objspaceclass'    : 'space.newtext(repr(space))',
-        #'prefix'               : # added by pypy_initial_path() when it
-        #'exec_prefix'          : # succeeds, pointing to trunk or /usr
+        'prefix'                : 'state.get(space).w_initial_prefix',
+        'exec_prefix'           : 'state.get(space).w_initial_prefix',
         'path'                  : 'state.get(space).w_path',
         'modules'               : 'state.get(space).w_modules',
         'argv'                  : 'state.get(space).w_argv',
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -19,8 +19,11 @@
 
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
+        # This initial value for sys.prefix is normally overwritten
+        # at runtime by initpath.py
+        srcdir = os.path.dirname(pypydir)
+        self.w_initial_prefix = space.newtext(srcdir)
         # Initialize the default path
-        srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.newtext(p) for p in path])
 

From pypy.commits at gmail.com  Sun May 28 13:36:57 2017
From: pypy.commits at gmail.com (planter)
Date: Sun, 28 May 2017 10:36:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve zipfile performance by not
 doing repeated string concatenation
Message-ID: <592b0ab9.32b0df0a.f87e.f404@mx.google.com>

Author: Petre Vijiac <petre.vijiac at rinftech.com>
Branch: 
Changeset: r91434:bed189ed5900
Date: 2017-05-28 19:35 +0200
http://bitbucket.org/pypy/pypy/changeset/bed189ed5900/

Log:	improve zipfile performance by not doing repeated string
	concatenation

	(committed by cfbolz)

diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/2.7/zipfile.py
@@ -622,19 +622,23 @@
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached..
         """
-        buf = ''
+        # PyPy modification: don't do repeated string concatenation
+        buf = []
+        lenbuf = 0
         if n is None:
             n = -1
         while True:
             if n < 0:
                 data = self.read1(n)
-            elif n > len(buf):
-                data = self.read1(n - len(buf))
+            elif n > lenbuf:
+                data = self.read1(n - lenbuf)
             else:
-                return buf
+                break
             if len(data) == 0:
-                return buf
-            buf += data
+                break
+            lenbuf += len(data)
+            buf.append(data)
+        return "".join(buf)
 
     def _update_crc(self, newdata, eof):
         # Update the CRC using the given data.

From pypy.commits at gmail.com  Mon May 29 04:27:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 01:27:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add checks on the length argument given
 to socket.getsockopt(), like CPython
Message-ID: <592bdb8f.54b21c0a.c19ad.8dd2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91435:b88087dc021a
Date: 2017-05-29 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/b88087dc021a/

Log:	Add checks on the length argument given to socket.getsockopt(), like
	CPython

diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -310,6 +310,8 @@
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
+        if buflen < 0 or buflen > 1024:
+            raise explicit_socket_error(space, "getsockopt buflen out of range")
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
@@ -696,6 +698,12 @@
         w_exception = space.call_function(w_exception_class, space.newtext(message))
     return OperationError(w_exception_class, w_exception)
 
+def explicit_socket_error(space, msg):
+    w_exception_class = space.fromcache(SocketAPI).w_error
+    w_exception = space.call_function(w_exception_class, space.newtext(msg))
+    return OperationError(w_exception_class, w_exception)
+
+
 # ____________________________________________________________
 
 socketmethodnames = """
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -595,6 +595,16 @@
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
 
+    def test_getsockopt_bad_length(self):
+        import _socket
+        s = _socket.socket()
+        buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
+        assert buf == b'\x00' * 4
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, -1)
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':

From pypy.commits at gmail.com  Mon May 29 12:04:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 09:04:33 -0700 (PDT)
Subject: [pypy-commit] cffi default: tweaks
Message-ID: <592c4691.05421c0a.62f92.6590@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2933:93fa52b7eed3
Date: 2017-05-29 18:04 +0200
http://bitbucket.org/cffi/cffi/changeset/93fa52b7eed3/

Log:	tweaks

diff --git a/demo/xclient_build.py b/demo/xclient_build.py
--- a/demo/xclient_build.py
+++ b/demo/xclient_build.py
@@ -22,4 +22,4 @@
 """, libraries=['X11'])
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffi.compile(verbose=True)
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -73,8 +73,8 @@
 ++++++++++++++++++++++++++++
 
 **ffi.errno**: the value of ``errno`` received from the most recent C call
-in this thread, and passed to the following C call.  (This is a read-write
-property.)
+in this thread, and passed to the following C call.  (This is a thread-local
+read-write property.)
 
 **ffi.getwinerror(code=-1)**: on Windows, in addition to ``errno`` we
 also save and restore the ``GetLastError()`` value across function

From pypy.commits at gmail.com  Mon May 29 13:11:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:11:20 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Fix test, fix all abort()
Message-ID: <592c5638.11addf0a.2b75b.f215@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2936:ab6edb3987e7
Date: 2017-05-29 18:50 +0200
http://bitbucket.org/cffi/cffi/changeset/ab6edb3987e7/

Log:	Fix test, fix all abort()

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1565,7 +1565,11 @@
         return convert_struct_from_object(data, ct, init, NULL);
     }
     if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
-        abort(); // XXX
+        Py_complex value = PyComplex_AsCComplex(init);
+        if (PyErr_Occurred())
+            return -1;
+        write_raw_complex_data(data, value, ct->ct_size);
+        return 0;
     }
     PyErr_Format(PyExc_SystemError,
                  "convert_from_object: '%s'", ct->ct_name);
@@ -2005,7 +2009,9 @@
             return read_raw_float_data(cd->c_data, cd->c_type->ct_size) != 0.0;
         }
         if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
-            abort(); // XXX
+            Py_complex value = read_raw_complex_data(cd->c_data,
+                                                     cd->c_type->ct_size);
+            return value.real != 0.0 || value.imag != 0.0;
         }
     }
     return cd->c_data != NULL;
@@ -2964,19 +2970,10 @@
         PyObject *op = PyComplex_FromCComplex(value);
         return op;
     }
-    // floats can also be converted to complex
-    if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
-        Py_complex value;
-        if (!(cd->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
-            value.real = read_raw_float_data(cd->c_data, cd->c_type->ct_size);
-        }
-        else {
-            value.real = (double)read_raw_longdouble_data(cd->c_data);
-        }
-        value.imag = 0.0;
-        return PyComplex_FromCComplex(value);
-    }
-    abort(); // XXX ints, etc.
+    /* <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+       calling complex(), just like <cdata 'int'> cannot be directly
+       converted by calling float() */
+
     PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
                  cd->c_type->ct_name);
     return NULL;
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -217,16 +217,17 @@
         #
         py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
         #
-        assert complex(cast(new_primitive_type("char"), "A")) == 65 + 0j
-        assert complex(cast(new_primitive_type("int"), 65)) == 65 + 0j
-        assert complex(cast(new_primitive_type("uint64_t"), 65)) == 65 + 0j
-        assert complex(cast(new_primitive_type("float"), 65.5)) == 65.5 + 0j
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
         #
         BArray = new_array_type(new_pointer_type(p), 10)
         x = newp(BArray, None)
         x[5] = 12.34 + 56.78j
         assert type(x[5]) is complex
-        assert x[5] == 12.34 + 56.78j
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
     py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():

From pypy.commits at gmail.com  Mon May 29 13:11:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:11:16 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: hg merge default
Message-ID: <592c5634.44bf1c0a.68b41.d859@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2934:5e5d1029dab9
Date: 2017-05-29 18:11 +0200
http://bitbucket.org/cffi/cffi/changeset/5e5d1029dab9/

Log:	hg merge default

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/cffi/error.py b/cffi/error.py
--- a/cffi/error.py
+++ b/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1479,6 +1479,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/cffi/verifier.py b/cffi/verifier.py
--- a/cffi/verifier.py
+++ b/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/demo/xclient_build.py b/demo/xclient_build.py
--- a/demo/xclient_build.py
+++ b/demo/xclient_build.py
@@ -22,4 +22,4 @@
 """, libraries=['X11'])
 
 if __name__ == '__main__':
-    ffi.compile()
+    ffi.compile(verbose=True)
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -248,6 +248,16 @@
         );
       """))
 
+Note also that pycparser, the underlying C parser, recognizes
+preprocessor-like directives in the following format: ``# NUMBER
+"FILE"``.  For example, if you put ``# 42 "foo.h"`` in the middle of the
+string passed to ``cdef()`` and there is an error two lines later, then
+it is reported with an error message that starts with ``foo.h:43:`` (the
+line which is given the number 42 is the line immediately after the
+directive).  *New in version 1.10.1:*  CFFI automatically puts the line
+``# 1 "<cdef source string>"`` just before the string you give to
+``cdef()``.
+
 
 .. _`ffi.set_unicode()`:
 
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -73,8 +73,8 @@
 ++++++++++++++++++++++++++++
 
 **ffi.errno**: the value of ``errno`` received from the most recent C call
-in this thread, and passed to the following C call.  (This is a read-write
-property.)
+in this thread, and passed to the following C call.  (This is a thread-local
+read-write property.)
 
 **ffi.getwinerror(code=-1)**: on Windows, in addition to ``errno`` we
 also save and restore the ``GetLastError()`` value across function
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,15 @@
 ======================
 
 
+v1.10.1
+=======
+
+* Fixed the line numbers reported in case of ``cdef()`` errors.
+  Also, I just noticed, but pycparser always supported the preprocessor
+  directive ``# 42 "foo.h"`` to mean "from the next line, we're in file
+  foo.h starting from line 42", which it puts in the error messages.
+
+
 v1.10
 =====
 
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -228,13 +228,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -278,7 +292,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -436,3 +451,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')

From pypy.commits at gmail.com  Mon May 29 13:11:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:11:18 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Minor style clean-ups.
 Update the test to conform more closely to the
Message-ID: <592c5636.493e1c0a.68e4f.185f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2935:9a8456642302
Date: 2017-05-29 18:36 +0200
http://bitbucket.org/cffi/cffi/changeset/9a8456642302/

Log:	Minor style clean-ups. Update the test to conform more closely to
	the newer version of test_float_types(). Add a few abort() in the
	source for places missing complex support.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -62,8 +62,6 @@
 
 #include "malloc_closure.h"
 
-#include <complex.h>
-
 #if PY_MAJOR_VERSION >= 3
 # define STR_OR_BYTES "bytes"
 # define PyText_Type PyUnicode_Type
@@ -118,34 +116,33 @@
 /************************************************************/
 
 /* base type flag: exactly one of the following: */
-#define CT_PRIMITIVE_SIGNED   1    /* signed integer */
-#define CT_PRIMITIVE_UNSIGNED 2    /* unsigned integer */
-#define CT_PRIMITIVE_CHAR     4    /* char, wchar_t */
-#define CT_PRIMITIVE_FLOAT    8    /* float, double, long double */
-#define CT_POINTER           16    /* pointer, excluding ptr-to-func */
-#define CT_ARRAY             32    /* array */
-#define CT_STRUCT            64    /* struct */
-#define CT_UNION            128    /* union */
-#define CT_FUNCTIONPTR      256    /* pointer to function */
-#define CT_VOID             512    /* void */
-
-#define CT_PRIMITIVE_COMPLEX 16777216  /* float _Complex, double _Complex */
+#define CT_PRIMITIVE_SIGNED   0x001   /* signed integer */
+#define CT_PRIMITIVE_UNSIGNED 0x002   /* unsigned integer */
+#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t */
+#define CT_PRIMITIVE_FLOAT    0x008   /* float, double, long double */
+#define CT_POINTER            0x010   /* pointer, excluding ptr-to-func */
+#define CT_ARRAY              0x020   /* array */
+#define CT_STRUCT             0x040   /* struct */
+#define CT_UNION              0x080   /* union */
+#define CT_FUNCTIONPTR        0x100   /* pointer to function */
+#define CT_VOID               0x200   /* void */
+#define CT_PRIMITIVE_COMPLEX  0x400   /* float _Complex, double _Complex */
 
 /* other flags that may also be set in addition to the base flag: */
-#define CT_IS_VOIDCHAR_PTR       1024
-#define CT_PRIMITIVE_FITS_LONG   2048
-#define CT_IS_OPAQUE             4096
-#define CT_IS_ENUM               8192
-#define CT_IS_PTR_TO_OWNED      16384   /* only owned if CDataOwning_Type */
-#define CT_CUSTOM_FIELD_POS     32768
-#define CT_IS_LONGDOUBLE        65536
-#define CT_IS_BOOL             131072
-#define CT_IS_FILE             262144
-#define CT_IS_VOID_PTR         524288
-#define CT_WITH_VAR_ARRAY     1048576
-#define CT_IS_UNSIZED_CHAR_A  2097152
-#define CT_LAZY_FIELD_LIST    4194304
-#define CT_WITH_PACKED_CHANGE 8388608
+#define CT_IS_VOIDCHAR_PTR     0x00001000
+#define CT_PRIMITIVE_FITS_LONG 0x00002000
+#define CT_IS_OPAQUE           0x00004000
+#define CT_IS_ENUM             0x00008000
+#define CT_IS_PTR_TO_OWNED     0x00010000 /* only owned if CDataOwning_Type */
+#define CT_CUSTOM_FIELD_POS    0x00020000
+#define CT_IS_LONGDOUBLE       0x00040000
+#define CT_IS_BOOL             0x00080000
+#define CT_IS_FILE             0x00100000
+#define CT_IS_VOID_PTR         0x00200000
+#define CT_WITH_VAR_ARRAY      0x00400000
+#define CT_IS_UNSIZED_CHAR_A   0x00800000
+#define CT_LAZY_FIELD_LIST     0x01000000
+#define CT_WITH_PACKED_CHANGE  0x02000000
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
@@ -1567,6 +1564,9 @@
         }
         return convert_struct_from_object(data, ct, init, NULL);
     }
+    if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        abort(); // XXX
+    }
     PyErr_Format(PyExc_SystemError,
                  "convert_from_object: '%s'", ct->ct_name);
     return -1;
@@ -2004,6 +2004,9 @@
                 return read_raw_longdouble_data(cd->c_data) != 0.0;
             return read_raw_float_data(cd->c_data, cd->c_type->ct_size) != 0.0;
         }
+        if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
+            abort(); // XXX
+        }
     }
     return cd->c_data != NULL;
 }
@@ -2072,11 +2075,6 @@
         }
         return PyFloat_FromDouble(value);
     }
-    if ((cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) &&
-        (cd->c_type->ct_size<16)) {
-        double value = read_raw_float_data(cd->c_data, cd->c_type->ct_size);
-        return PyComplex_FromDoubles(value, 0.0);
-    }
     PyErr_Format(PyExc_TypeError, "float() not supported on cdata '%s'",
                  cd->c_type->ct_name);
     return NULL;
@@ -2978,6 +2976,7 @@
         value.imag = 0.0;
         return PyComplex_FromCComplex(value);
     }
+    abort(); // XXX ints, etc.
     PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
                  cd->c_type->ct_name);
     return NULL;
@@ -3792,8 +3791,6 @@
         }
         return (PyObject *)cd;
     }
-
-
     else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
         /* cast to a complex */
         Py_complex value;
@@ -3835,8 +3832,6 @@
         }
         return (PyObject *)cd;
     }
-
-
     else {
         PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
                      ct->ct_name);
@@ -6738,6 +6733,8 @@
         return 1000 * p[0];
     return -42;
 }
+
+#if 0   /* libffi doesn't properly support complexes currently */
 static float _Complex _testfunc24(float a, float b)
 {
     return a + I*2.0*b;
@@ -6746,6 +6743,7 @@
 {
     return a + I*2.0*b;
 }
+#endif
 
 static PyObject *b__testfunc(PyObject *self, PyObject *args)
 {
@@ -6779,8 +6777,10 @@
     case 21: f = &_testfunc21; break;
     case 22: f = &_testfunc22; break;
     case 23: f = &_testfunc23; break;
+#if 0
     case 24: f = &_testfunc24; break;
     case 25: f = &_testfunc25; break;
+#endif
     default:
         PyErr_SetNone(PyExc_ValueError);
         return NULL;
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -188,12 +188,13 @@
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
         p = new_primitive_type(name + " _Complex")
-        assert bool(cast(p, 0))
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
@@ -209,13 +210,24 @@
         assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
         #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
-        assert complex(cast(p, b'\x09')) == 9.0
-        assert complex(cast(p, u+'\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(TypeError, cast, new_primitive_type(name), 1+2j)
-    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        assert complex(cast(new_primitive_type("char"), "A")) == 65 + 0j
+        assert complex(cast(new_primitive_type("int"), 65)) == 65 + 0j
+        assert complex(cast(new_primitive_type("uint64_t"), 65)) == 65 + 0j
+        assert complex(cast(new_primitive_type("float"), 65.5)) == 65.5 + 0j
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert x[5] == 12.34 + 56.78j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")

From pypy.commits at gmail.com  Mon May 29 13:11:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:11:21 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Test explicitly that if
 we try to call a function in ABI mode with a
Message-ID: <592c5639.06a81c0a.8dba7.8266@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2937:f0f90cb166e5
Date: 2017-05-29 19:10 +0200
http://bitbucket.org/cffi/cffi/changeset/f0f90cb166e5/

Log:	Test explicitly that if we try to call a function in ABI mode with a
	complex argument or return value, then we get a clear error message.
	Previously, it would still try to call libffi. But libffi doesn't
	support complex, on all platforms except a single very obscure one,
	I think. So it seems to be a better behaviour for now.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -888,7 +888,7 @@
 static Py_complex
 read_raw_complex_data(char *target, int size)
 {
-    Py_complex r = {.real=0, .imag=0};
+    Py_complex r = {0.0, 0.0};
     if (size == 2*sizeof(float)) {
         float real_part, imag_part;
         memcpy(&real_part, target + 0,             sizeof(float));
@@ -898,7 +898,7 @@
         return r;
     }
     if (size == 2*sizeof(double)) {
-        memcpy(&(r.real), target, 2*sizeof(double));
+        memcpy(&r, target, 2*sizeof(double));
         return r;
     }
     Py_FatalError("read_raw_complex_data: bad float size");
@@ -4224,14 +4224,11 @@
             goto bad_ffi_type;
     }
     else if (ptypes->flags & CT_PRIMITIVE_COMPLEX) {
-        // as of March 2017, still no libffi support for complex
-        // but it fails silently.
-        if (strcmp(ptypes->name, "float _Complex") == 0)
-            ffitype = &ffi_type_complex_float;
-        else if (strcmp(ptypes->name, "double _Complex") == 0)
-            ffitype = &ffi_type_complex_double;
-        else
-            goto bad_ffi_type;
+        /* As of March 2017, still no libffi support for complex.
+           It fails silently if we try to use ffi_type_complex_float
+           or ffi_type_complex_double.  Better not use it at all.
+         */
+        ffitype = NULL;
     }
     else {
         switch (ptypes->size) {
@@ -4925,7 +4922,7 @@
 {
     const char *place = is_result_type ? "return value" : "argument";
 
-    if (ct->ct_flags & CT_PRIMITIVE_ANY) {
+    if (ct->ct_flags & (CT_PRIMITIVE_ANY & ~CT_PRIMITIVE_COMPLEX)) {
         return (ffi_type *)ct->ct_extra;
     }
     else if (ct->ct_flags & (CT_POINTER|CT_FUNCTIONPTR)) {
@@ -5051,9 +5048,16 @@
         return NULL;
     }
     else {
+        char *extra = "";
+        if (ct->ct_flags & CT_PRIMITIVE_COMPLEX)
+            extra = " (the support for complex types inside libffi "
+                    "is mostly missing at this point, so CFFI only "
+                    "supports complex types as arguments or return "
+                    "value in API-mode functions)";
+
         PyErr_Format(PyExc_NotImplementedError,
-                     "ctype '%s' (size %zd) not supported as %s",
-                     ct->ct_name, ct->ct_size, place);
+                     "ctype '%s' (size %zd) not supported as %s%s",
+                     ct->ct_name, ct->ct_size, place, extra);
         return NULL;
     }
 }
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1130,27 +1130,32 @@
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
 def test_call_function_24():
-    py.test.skip("libffi returning nonsense silently")
     BFloat = new_primitive_type("float")
     BFloatComplex = new_primitive_type("float _Complex")
     BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
-    f = cast(BFunc3, _testfunc(24))
-    result = f(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact  
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
 
 def test_call_function_25():
-    py.test.skip("libffi returning nonsense silently")
     BDouble = new_primitive_type("double")
     BDoubleComplex = new_primitive_type("double _Complex")
     BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
-    f = cast(BFunc3, _testfunc(25))
-    result = f(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact  
-
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
 
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")

From pypy.commits at gmail.com  Mon May 29 13:29:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:29:33 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Tweaks
Message-ID: <592c5a7d.eb8edf0a.460a.5d42@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2938:715175718364
Date: 2017-05-29 19:22 +0200
http://bitbucket.org/cffi/cffi/changeset/715175718364/

Log:	Tweaks

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -901,7 +901,7 @@
         memcpy(&r, target, 2*sizeof(double));
         return r;
     }
-    Py_FatalError("read_raw_complex_data: bad float size");
+    Py_FatalError("read_raw_complex_data: bad complex size");
     return r;
 }
 
@@ -936,7 +936,7 @@
 {
     _write_raw_complex_data(float);
     _write_raw_complex_data(double);
-    Py_FatalError("write_raw_complex_data: bad float size");
+    Py_FatalError("write_raw_complex_data: bad complex size");
 }
 
 static PyObject *
@@ -1047,10 +1047,6 @@
             return (PyObject *)cd;
         }
     }
-    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
-        Py_complex value = read_raw_complex_data(data, ct->ct_size);
-        return PyComplex_FromCComplex(value);
-    }
     else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(data, ct->ct_size)*/
         if (ct->ct_size == sizeof(char))
@@ -1060,6 +1056,10 @@
             return _my_PyUnicode_FromWideChar((wchar_t *)data, 1);
 #endif
     }
+    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = read_raw_complex_data(data, ct->ct_size);
+        return PyComplex_FromCComplex(value);
+    }
 
     PyErr_Format(PyExc_SystemError,
                  "convert_to_object: '%s'", ct->ct_name);
@@ -3663,34 +3663,29 @@
     return cd;
 }
 
-static void check_bytes_for_float_compatible(
-        PyObject *io,
-        double * value,
-        int * got_value_indicator,
-        int * cannot_cast_indicator)
-{
-    *got_value_indicator = 0;
-    *cannot_cast_indicator = 0;
+/* returns -1 if cannot cast, 0 if we don't get a value, 1 if we do */
+static int check_bytes_for_float_compatible(PyObject *io, double *out_value)
+{
     if (PyBytes_Check(io)) {
         if (PyBytes_GET_SIZE(io) != 1) {
             Py_DECREF(io);
-            *cannot_cast_indicator = 1;
-        }
-        *value = (unsigned char)PyBytes_AS_STRING(io)[0];
-        *got_value_indicator = 1;
+            return -1;
+        }
+        *out_value = (unsigned char)PyBytes_AS_STRING(io)[0];
+        return 1;
     }
 #if HAVE_WCHAR_H
     else if (PyUnicode_Check(io)) {
         wchar_t ordinal;
         if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
             Py_DECREF(io);
-            *cannot_cast_indicator = 1;
-        }
-        *value = (long)ordinal;
-        *got_value_indicator = 1;
+            return -1;
+        }
+        *out_value = (long)ordinal;
+        return 1;
     }
 #endif
-
+    return 0;
 }
 
 static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
@@ -3749,30 +3744,23 @@
             Py_INCREF(io);
         }
 
-        int got_value_indicator;
-        int cannot_cast_indicator;
-        check_bytes_for_float_compatible(io, &value,
-                &got_value_indicator, &cannot_cast_indicator);
-        if (cannot_cast_indicator) {
+        int res = check_bytes_for_float_compatible(io, &value);
+        if (res == -1)
             goto cannot_cast;
-        }
-        if (got_value_indicator) {
-            // got it from string
-        }
-        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+        if (res == 0) {
+            if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
                  CData_Check(io) &&
                  (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
-            long double lvalue;
-            char *data = ((CDataObject *)io)->c_data;
-            /*READ(data, sizeof(long double)*/
-            lvalue = read_raw_longdouble_data(data);
-            Py_DECREF(io);
-            cd = _new_casted_primitive(ct);
-            if (cd != NULL)
-                write_raw_longdouble_data(cd->c_data, lvalue);
-            return (PyObject *)cd;
-        }
-        else {
+                long double lvalue;
+                char *data = ((CDataObject *)io)->c_data;
+                /*READ(data, sizeof(long double)*/
+                lvalue = read_raw_longdouble_data(data);
+                Py_DECREF(io);
+                cd = _new_casted_primitive(ct);
+                if (cd != NULL)
+                    write_raw_longdouble_data(cd->c_data, lvalue);
+                return (PyObject *)cd;
+            }
             value = PyFloat_AsDouble(io);
         }
         Py_DECREF(io);
@@ -3806,21 +3794,17 @@
             Py_INCREF(io);
         }
 
-        int got_value_indicator;
-        int cannot_cast_indicator;
-        check_bytes_for_float_compatible(io, &(value.real),
-                &got_value_indicator, &cannot_cast_indicator);
-        if (cannot_cast_indicator) {
+        int res = check_bytes_for_float_compatible(io, &value.real);
+        if (res == -1)
             goto cannot_cast;
-        }
-        if (got_value_indicator) {
+        if (res == 1) {
             // got it from string
             value.imag = 0.0;
         } else {
             value = PyComplex_AsCComplex(io);
         }
         Py_DECREF(io);
-        if (value.real == -1.0 && value.imag == 0.0 && PyErr_Occurred()) {
+        if (PyErr_Occurred()) {
             return NULL;
         }
         cd = _new_casted_primitive(ct);

From pypy.commits at gmail.com  Mon May 29 13:29:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:29:35 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: MSVC might not support
 _Complex (and the include <complex.h>).
Message-ID: <592c5a7f.91d81c0a.27ba7.ee2f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2939:c7a53457ef79
Date: 2017-05-29 19:29 +0200
http://bitbucket.org/cffi/cffi/changeset/c7a53457ef79/

Log:	MSVC might not support _Complex (and the include <complex.h>).

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -7,6 +7,8 @@
 #ifdef MS_WIN32
 #include <windows.h>
 #include "misc_win32.h"
+typedef float cffi_float_complex_t[2];
+typedef double cffi_double_complex_t[2];
 #else
 #include <stddef.h>
 #include <stdint.h>
@@ -14,6 +16,8 @@
 #include <errno.h>
 #include <ffi.h>
 #include <sys/mman.h>
+typedef float _Complex cffi_float_complex_t; /* use the def above if your C */
+typedef double _Complex cffi_double_complex_t; /*   compiler complains here */
 #endif
 
 /* this block of #ifs should be kept exactly identical between
@@ -4095,8 +4099,8 @@
        EPTYPE(f, float, CT_PRIMITIVE_FLOAT )                    \
        EPTYPE(d, double, CT_PRIMITIVE_FLOAT )                   \
        EPTYPE(ld, long double, CT_PRIMITIVE_FLOAT | CT_IS_LONGDOUBLE ) \
-       EPTYPE(fc, float _Complex, CT_PRIMITIVE_COMPLEX )        \
-       EPTYPE(dc, double _Complex, CT_PRIMITIVE_COMPLEX )       \
+       EPTYPE2(fc, "float _Complex", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX ) \
+       EPTYPE2(dc, "double _Complex", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX ) \
        ENUM_PRIMITIVE_TYPES_WCHAR                               \
        EPTYPE(b, _Bool, CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL )    \
      /* the following types are not primitive in the C sense */ \
@@ -6720,6 +6724,7 @@
 }
 
 #if 0   /* libffi doesn't properly support complexes currently */
+        /* also, MSVC might not support _Complex... */
 static float _Complex _testfunc24(float a, float b)
 {
     return a + I*2.0*b;

From pypy.commits at gmail.com  Mon May 29 13:31:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:31:55 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: comment
Message-ID: <592c5b0b.15371c0a.79ce5.bd4b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2940:ecf49f8b034c
Date: 2017-05-29 19:31 +0200
http://bitbucket.org/cffi/cffi/changeset/ecf49f8b034c/

Log:	comment

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6725,6 +6725,8 @@
 
 #if 0   /* libffi doesn't properly support complexes currently */
         /* also, MSVC might not support _Complex... */
+        /* if this is enabled one day, remember to also add _Complex
+         * arguments in addition to return values. */
 static float _Complex _testfunc24(float a, float b)
 {
     return a + I*2.0*b;

From pypy.commits at gmail.com  Mon May 29 13:52:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:52:14 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Revert the changes to
 vengine_cpy: better to say that the
Message-ID: <592c5fce.3098df0a.f3c63.04b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2941:de5953e587b8
Date: 2017-05-29 19:35 +0200
http://bitbucket.org/cffi/cffi/changeset/de5953e587b8/

Log:	Revert the changes to vengine_cpy: better to say that the long-
	deprecated verify mode doesn't support complex

diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -806,7 +806,6 @@
 cffimod_header = r'''
 #include <Python.h>
 #include <stddef.h>
-#include <complex.h>
 
 /* this block of #ifs should be kept exactly identical between
    c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
@@ -874,29 +873,6 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
-#define _cffi_from_c_float__Complex(x)  PyComplex_FromDoubles(crealf(x), cimagf(x))
-#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x))
-
-/* inefficient - converts twice! */
-#define _cffi_to_c_float__Complex(op)                                    \
-    (  ((float)(PyComplex_AsCComplex(op).real))  +                       \
-     I*((float)(PyComplex_AsCComplex(op).imag))     )
-/* not safe!
-//#define _cffi_to_c_float__Complex(op)                                  \
-//    (  ((float)((PyComplexObject *)(op))->cval.real)  +                \
-//     I*((float)((PyComplexObject *)(op))->cval.imag)     )
-*/
-
-/* inefficient - converts twice! */
-#define _cffi_to_c_double__Complex(op)                                   \
-    (  (PyComplex_AsCComplex(op).real)  +                                \
-     I*(PyComplex_AsCComplex(op).imag)     )
-/* not safe!
-//#define _cffi_to_c_double__Complex(op)                                   \
-//    (  (((PyComplexObject *)(op))->cval.real)  +                         \
-//     I*(((PyComplexObject *)(op))->cval.imag)     )
-*/
-
 #define _cffi_from_c_int_const(x)                                        \
     (((x) > 0) ?                                                         \
         ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ?      \

From pypy.commits at gmail.com  Mon May 29 13:52:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 10:52:16 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Support parsing "float
 _Complex" and "double _Complex" inside parse_c_type
Message-ID: <592c5fd0.4c3a1c0a.7e266.43ef@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2942:f6e7adb9a3b5
Date: 2017-05-29 19:52 +0200
http://bitbucket.org/cffi/cffi/changeset/f6e7adb9a3b5/

Log:	Support parsing "float _Complex" and "double _Complex" inside
	parse_c_type

diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -159,6 +159,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -601,6 +602,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -656,6 +658,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -666,6 +670,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -719,9 +724,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -788,6 +795,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok,"_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/testing/cffi1/test_parse_c_type.py b/testing/cffi1/test_parse_c_type.py
--- a/testing/cffi1/test_parse_c_type.py
+++ b/testing/cffi1/test_parse_c_type.py
@@ -155,6 +155,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -280,6 +282,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize

From pypy.commits at gmail.com  Mon May 29 14:37:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 11:37:18 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: More tweaks, more tests,
 try harder to avoid including <complex.h>
Message-ID: <592c6a5e.88da1c0a.c7b8.9ca0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2943:117488cc30fc
Date: 2017-05-29 20:37 +0200
http://bitbucket.org/cffi/cffi/changeset/117488cc30fc/

Log:	More tweaks, more tests, try harder to avoid including <complex.h>
	because it is not necessarily there

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -7,8 +7,6 @@
 #ifdef MS_WIN32
 #include <windows.h>
 #include "misc_win32.h"
-typedef float cffi_float_complex_t[2];
-typedef double cffi_double_complex_t[2];
 #else
 #include <stddef.h>
 #include <stdint.h>
@@ -16,8 +14,6 @@
 #include <errno.h>
 #include <ffi.h>
 #include <sys/mman.h>
-typedef float _Complex cffi_float_complex_t; /* use the def above if your C */
-typedef double _Complex cffi_double_complex_t; /*   compiler complains here */
 #endif
 
 /* this block of #ifs should be kept exactly identical between
@@ -4082,6 +4078,11 @@
     return NULL;
 }
 
+/* according to the C standard, these types should be equivalent to the
+   _Complex types for the purposes of storage (not arguments in calls!) */
+typedef float cffi_float_complex_t[2];
+typedef double cffi_double_complex_t[2];
+
 static PyObject *new_primitive_type(const char *name)
 {
 #define ENUM_PRIMITIVE_TYPES                                    \
diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -798,7 +798,7 @@
     if (tok->kind == TOK__COMPLEX)
     {
         if (t1complex == 0)
-            return parse_error(tok,"_Complex type combination unsupported");
+            return parse_error(tok, "_Complex type combination unsupported");
         t1 = t1complex;
         next_token(tok);
     }
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -16,7 +16,6 @@
 #endif
 
 #include <Python.h>
-#include <complex.h>
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -100,29 +99,6 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
-#define _cffi_from_c_float__Complex(x)  PyComplex_FromDoubles(crealf(x), cimagf(x))
-#define _cffi_from_c_double__Complex(x) PyComplex_FromDoubles(creal(x), cimag(x))
-
-/* inefficient - converts twice! */
-#define _cffi_to_c_float__Complex(op)                                    \
-   (  ((float)(PyComplex_AsCComplex(op).real))  +                       \
-    I*((float)(PyComplex_AsCComplex(op).imag))     )
-/* not safe!
-//#define _cffi_to_c_float__Complex(op)                                  \
-//    (  ((float)((PyComplexObject *)(op))->cval.real)  +                \
-//     I*((float)((PyComplexObject *)(op))->cval.imag)     )
-*/
-
-/* inefficient - converts twice! */
-#define _cffi_to_c_double__Complex(op)                                   \
-   (  (PyComplex_AsCComplex(op).real)  +                                \
-    I*(PyComplex_AsCComplex(op).imag)     )
-/* not safe!
-//#define _cffi_to_c_double__Complex(op)                                   \
-//    (  (((PyComplexObject *)(op))->cval.real)  +                         \
-//     I*(((PyComplexObject *)(op))->cval.imag)     )
-*/
-
 #define _cffi_from_c_int(x, type)                                        \
     (((type)-1) > 0 ? /* unsigned */                                     \
         (sizeof(type) < sizeof(long) ?                                   \
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -116,8 +116,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
-        'float _Complex':     'f',
-        'double _Complex':    'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -165,6 +165,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -239,17 +239,19 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
-        assert F == (typename in ('float', 'double', 'long double', 'float _Complex', 'double _Complex'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert F == (typename in ('float', 'double', 'long double'))
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            typename == '_Bool' or typename == 'long double'
-            or '_Complex' in typename):  # omit _Complex since ffi does not yet support
+            all_primitive_types[typename] == 'j' or    # complex
+            typename == '_Bool' or typename == 'long double'):
             pass
         else:
             typenames.append(typename)
diff --git a/testing/cffi1/test_realize_c_type.py b/testing/cffi1/test_realize_c_type.py
--- a/testing/cffi1/test_realize_c_type.py
+++ b/testing/cffi1/test_realize_c_type.py
@@ -45,14 +45,7 @@
 
 def test_all_primitives():
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
-        if '_Complex' not in name:
-            check(name, name)
-
-def test_complex_primitives():
-    py.test.xfail("ffi does not support complex yet")
-    for name in cffi_opcode.PRIMITIVE_TO_INDEX:
-        if '_Complex' in name:
-            check(name, name)
+        check(name, name)
 
 def check_func(input, expected_output=None):
     import _cffi_backend
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2002,28 +2002,58 @@
     assert lib.f1(52).a == 52
 
 def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
     ffi = FFI()
     ffi.cdef("float _Complex f1(float a, float b);");
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
-        static float _Complex f1 (float a, float b) { return a + I*2.0*b; }
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
     """)
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
-    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact  
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
 
 def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
     ffi = FFI()
     ffi.cdef("double _Complex f1(double a, double b);");
     lib = verify(ffi, "test_function_returns_double_complex", """
         #include <complex.h>
-        static double _Complex f1 (double a, double b) { return a + I*2.0*b; }
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
     """)
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
-    assert result.imag == 2*5.1  # exact  
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
 
 def test_typedef_array_dotdotdot():
     ffi = FFI()
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -219,15 +219,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
-        assert F == (typename in ('float', 'double', 'long double', 'float _Complex', 'double _Complex'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert F == (typename in ('float', 'double', 'long double'))
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Mon May 29 15:02:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 12:02:22 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: MSVC fixes
Message-ID: <592c703e.5497df0a.d11b5.4fcb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2945:c60f27f65b7b
Date: 2017-05-29 20:42 +0200
http://bitbucket.org/cffi/cffi/changeset/c60f27f65b7b/

Log:	MSVC fixes

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -924,8 +924,8 @@
     do {                                                   \
         if (size == 2*sizeof(type)) {                      \
             type r = (type)source.real;                    \
+            type i = (type)source.imag;                    \
             memcpy(target, &r, sizeof(type));              \
-            type i = (type)source.imag;                    \
             memcpy(target+sizeof(type), &i, sizeof(type)); \
             return;                                        \
         }                                                  \
@@ -3729,6 +3729,7 @@
         /* cast to a float */
         double value;
         PyObject *io;
+        int res;
 
         if (CData_Check(ob)) {
             CDataObject *cdsrc = (CDataObject *)ob;
@@ -3744,7 +3745,7 @@
             Py_INCREF(io);
         }
 
-        int res = check_bytes_for_float_compatible(io, &value);
+        res = check_bytes_for_float_compatible(io, &value);
         if (res == -1)
             goto cannot_cast;
         if (res == 0) {
@@ -3780,6 +3781,8 @@
         /* cast to a complex */
         Py_complex value;
         PyObject *io;
+        int res;
+
         if (CData_Check(ob)) {
             CDataObject *cdsrc = (CDataObject *)ob;
 
@@ -3794,7 +3797,7 @@
             Py_INCREF(io);
         }
 
-        int res = check_bytes_for_float_compatible(io, &value.real);
+        res = check_bytes_for_float_compatible(io, &value.real);
         if (res == -1)
             goto cannot_cast;
         if (res == 1) {

From pypy.commits at gmail.com  Mon May 29 15:02:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 29 May 2017 12:02:20 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: fix
Message-ID: <592c703c.b885df0a.cad2b.9184@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2944:7731e4b006ca
Date: 2017-05-29 20:40 +0200
http://bitbucket.org/cffi/cffi/changeset/7731e4b006ca/

Log:	fix

diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):

From pypy.commits at gmail.com  Mon May 29 17:37:08 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 May 2017 14:37:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: test, add a failure path,
 and document differences in PyTuple_SetItem
Message-ID: <592c9484.03541c0a.3b56f.2037@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91436:3e565506d1ed
Date: 2017-05-29 22:33 +0300
http://bitbucket.org/pypy/pypy/changeset/3e565506d1ed/

Log:	test, add a failure path, and document differences in
	PyTuple_SetItem

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -357,6 +357,25 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+C-API Differences
+-----------------
+
+The external C-API has been reimplemented in PyPy as an internal cpyext module.
+We support most of the documented C-API, but sometimes internal C-abstractions
+leak out on CPython and are abused, perhaps even unknowingly. For instance,
+assignment to a ``PyTupleObject`` is not supported on PyPy after the tuple is
+used internally, even if its refcount is 1 (whatever that means). On PyPy this
+will raise a ``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
+exception (explicitly listed here for search engines).
+
+Another similar problem is assignment of a new function pointer to any of the
+``tp_as_*`` structures after calling ``PyType_Ready``. For instance, overriding
+``tp_as_number.nb_int`` with a different function after calling ``PyType_Ready``
+on CPython will result in the old function being called for ``x.__int__()``
+(via class ``__dict__`` lookup) and the new function being called for ``int(x)``
+(via slot lookup). On PyPy we will always call the __new__ function, not the
+old, this quirky behaviour is unfortunately necessary to fully support NumPy.
+
 Performance Differences
 -------------------------
 
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -161,3 +161,38 @@
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
+
+    def test_setitem(self):
+        module = self.import_extension('foo', [
+            ("set_after_use", "METH_NOARGS",
+             """
+                PyObject *t2, *tuple = PyTuple_New(1);
+                PyObject * one = PyLong_FromLong(1);
+                Py_INCREF(one);
+                int res = PyTuple_SetItem(tuple, 0, one);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                /* Do something that uses the tuple, but does not incref */
+                t2 = PyTuple_GetSlice(tuple, 0, 1);
+                Py_DECREF(t2);
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                Py_DECREF(tuple);
+                if (res != 0)
+                {
+                    Py_DECREF(one);
+                    return NULL;
+                }
+                Py_INCREF(Py_None);
+                return Py_None;
+             """),
+            ])
+        import sys
+        if '__pypy__' in sys.builtin_module_names:
+            raises(SystemError, module.set_after_use)
+        else:
+            module.set_after_use()
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -6,7 +6,7 @@
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
-    track_reference, make_typedescr, get_typedescr)
+    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -132,9 +132,6 @@
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyTuple_SetItem(space, ref, index, py_obj):
-    # XXX this will not complain when changing tuples that have
-    # already been realized as a W_TupleObject, but won't update the
-    # W_TupleObject
     if not tuple_check_ref(space, ref):
         decref(space, py_obj)
         PyErr_BadInternalCall(space)
@@ -144,6 +141,10 @@
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
+    if old_ref and pyobj_has_w_obj(old_ref):
+        # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
+        raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
+                                        " use of tuple")
     ref.c_ob_item[index] = py_obj    # consumes a reference
     if old_ref:
         decref(space, old_ref)

From pypy.commits at gmail.com  Mon May 29 17:40:55 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 May 2017 14:40:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak documentation
Message-ID: <592c9567.505c1c0a.edbbb.2695@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91437:6cf91075579b
Date: 2017-05-30 00:40 +0300
http://bitbucket.org/pypy/pypy/changeset/6cf91075579b/

Log:	tweak documentation

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -363,9 +363,10 @@
 The external C-API has been reimplemented in PyPy as an internal cpyext module.
 We support most of the documented C-API, but sometimes internal C-abstractions
 leak out on CPython and are abused, perhaps even unknowingly. For instance,
-assignment to a ``PyTupleObject`` is not supported on PyPy after the tuple is
-used internally, even if its refcount is 1 (whatever that means). On PyPy this
-will raise a ``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
+assignment to a ``PyTupleObject`` is not supported after the tuple is
+used internally, even by another C-API function call. On CPython this will
+succeed as long as the refcount is 1.  On PyPy this will always raise a
+``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
 exception (explicitly listed here for search engines).
 
 Another similar problem is assignment of a new function pointer to any of the

From pypy.commits at gmail.com  Mon May 29 18:49:39 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 May 2017 15:49:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: check tuple, not contents
Message-ID: <592ca583.93921c0a.25989.b549@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91438:1049e523ffdb
Date: 2017-05-30 01:48 +0300
http://bitbucket.org/pypy/pypy/changeset/1049e523ffdb/

Log:	check tuple, not contents

diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -141,7 +141,7 @@
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
     old_ref = ref.c_ob_item[index]
-    if old_ref and pyobj_has_w_obj(old_ref):
+    if pyobj_has_w_obj(ref):
         # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
         raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
                                         " use of tuple")

From pypy.commits at gmail.com  Tue May 30 02:45:34 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 29 May 2017 23:45:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: expand test, fix translation
Message-ID: <592d150e.91471c0a.190c0.34e4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91439:b17c6b5755f2
Date: 2017-05-30 09:44 +0300
http://bitbucket.org/pypy/pypy/changeset/b17c6b5755f2/

Log:	expand test, fix translation

diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -164,12 +164,20 @@
 
     def test_setitem(self):
         module = self.import_extension('foo', [
-            ("set_after_use", "METH_NOARGS",
+            ("set_after_use", "METH_O",
              """
                 PyObject *t2, *tuple = PyTuple_New(1);
                 PyObject * one = PyLong_FromLong(1);
+                int res;
                 Py_INCREF(one);
-                int res = PyTuple_SetItem(tuple, 0, one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                Py_INCREF(args);
+                res = PyTuple_SetItem(tuple, 0, args);
                 if (res != 0)
                 {
                     Py_DECREF(tuple);
@@ -191,8 +199,9 @@
              """),
             ])
         import sys
+        s = 'abc'
         if '__pypy__' in sys.builtin_module_names:
-            raises(SystemError, module.set_after_use)
+            raises(SystemError, module.set_after_use, s)
         else:
-            module.set_after_use()
+            module.set_after_use(s)
 
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -135,17 +135,17 @@
     if not tuple_check_ref(space, ref):
         decref(space, py_obj)
         PyErr_BadInternalCall(space)
-    ref = rffi.cast(PyTupleObject, ref)
-    size = ref.c_ob_size
+    tupleobj = rffi.cast(PyTupleObject, ref)
+    size = tupleobj.c_ob_size
     if index < 0 or index >= size:
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
-    old_ref = ref.c_ob_item[index]
+    old_ref = tupleobj.c_ob_item[index]
     if pyobj_has_w_obj(ref):
         # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
         raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
                                         " use of tuple")
-    ref.c_ob_item[index] = py_obj    # consumes a reference
+    tupleobj.c_ob_item[index] = py_obj    # consumes a reference
     if old_ref:
         decref(space, old_ref)
     return 0

From pypy.commits at gmail.com  Tue May 30 07:37:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 04:37:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge jumbojet
Message-ID: <592d598b.91d81c0a.27ba7.d0d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91444:3ee9941e823c
Date: 2017-05-30 13:32 +0200
http://bitbucket.org/pypy/pypy/changeset/3ee9941e823c/

Log:	hg merge jumbojet

	sched_get min/max, by Joannah in PR #534

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -229,6 +229,14 @@
         'POSIX_FADV_RANDOM', 'POSIX_FADV_NOREUSE', 'POSIX_FADV_DONTNEED']:
             assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+    
+    if hasattr(rposix, 'sched_get_priority_max'):
+        interpleveldefs['sched_get_priority_max'] = 'interp_posix.sched_get_priority_max'
+        interpleveldefs['sched_get_priority_min'] = 'interp_posix.sched_get_priority_min'
+        for _name in ['SCHED_FIFO', 'SCHED_RR', 'SCHED_OTHER',
+        'SCHED_BATCH']:
+            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
+            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
 
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2418,3 +2418,31 @@
             except OSError as e:
                 wrap_oserror(space, e, eintr_retry=True)
     return space.newint(res)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_max(space, policy):
+    """returns the maximum priority value that 
+    can be used with the scheduling algorithm 
+    identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_max(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_min(space, policy):
+    """rreturns the minimum priority value that
+     can be used with the scheduling algorithm 
+     identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_min(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
\ No newline at end of file
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -937,6 +937,34 @@
             assert os.WIFEXITED(status1)
             assert os.WEXITSTATUS(status1) == 0   # else, test failure
 
+    if hasattr(rposix, 'sched_get_priority_max'):
+        def test_os_sched_get_priority_max(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_max(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_BATCH) != -1
+
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_get_priority_min(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_min(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_BATCH) != -1
+            
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_priority_max_greater_than_min(self):
+            posix, os = self.posix, self.os
+            policy = posix.SCHED_RR
+            low = posix.sched_get_priority_min(policy)
+            high = posix.sched_get_priority_max(policy)
+            assert isinstance(low, int) == True
+            assert isinstance(high, int) == True
+            assert  high > low
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -235,6 +235,7 @@
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'sys/resource.h',
+                'sched.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
@@ -255,6 +256,10 @@
     PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
     PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
     PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
+    SCHED_FIFO = rffi_platform.DefinedConstantInteger('SCHED_FIFO')
+    SCHED_RR = rffi_platform.DefinedConstantInteger('SCHED_RR')
+    SCHED_OTHER = rffi_platform.DefinedConstantInteger('SCHED_OTHER')
+    SCHED_BATCH = rffi_platform.DefinedConstantInteger('SCHED_BATCH')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -1818,6 +1823,21 @@
     def setpriority(which, who, prio):
         handle_posix_error('setpriority', c_setpriority(which, who, prio))
 
+    c_sched_get_priority_max = external('sched_get_priority_max', [rffi.INT],
+                              rffi.INT, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_sched_get_priority_min = external('sched_get_priority_min', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @enforceargs(int)
+    def sched_get_priority_max(policy):
+        return handle_posix_error('sched_get_priority_max', c_sched_get_priority_max(policy))
+
+    @enforceargs(int)
+    def sched_get_priority_min(policy):
+        return handle_posix_error('sched_get_priority_min', c_sched_get_priority_min(policy))
+
+
+
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -781,3 +781,27 @@
             raise
     finally:
         os.close(fd)
+
+ at rposix_requires('sched_get_priority_max')
+def test_sched_get_priority_max():
+    assert rposix.sched_get_priority_max(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_sched_get_priority_min():
+    assert rposix.sched_get_priority_min(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_os_sched_priority_max_greater_than_min():
+    policy = rposix.SCHED_RR
+    low = rposix.sched_get_priority_min(policy)
+    high = rposix.sched_get_priority_max(policy)
+    assert isinstance(low, int) == True
+    assert isinstance(high, int) == True
+    assert  high > low
+

From pypy.commits at gmail.com  Tue May 30 07:37:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 04:37:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge back changes to RPython in py3.5
 (3ee9941e823c)
Message-ID: <592d598d.2196df0a.cd98.6699@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91445:a65f0a6fd69b
Date: 2017-05-30 13:35 +0200
http://bitbucket.org/pypy/pypy/changeset/a65f0a6fd69b/

Log:	merge back changes to RPython in py3.5 (3ee9941e823c)

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -235,6 +235,7 @@
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'sys/resource.h',
+                'sched.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
@@ -255,6 +256,10 @@
     PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
     PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
     PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
+    SCHED_FIFO = rffi_platform.DefinedConstantInteger('SCHED_FIFO')
+    SCHED_RR = rffi_platform.DefinedConstantInteger('SCHED_RR')
+    SCHED_OTHER = rffi_platform.DefinedConstantInteger('SCHED_OTHER')
+    SCHED_BATCH = rffi_platform.DefinedConstantInteger('SCHED_BATCH')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -1818,6 +1823,21 @@
     def setpriority(which, who, prio):
         handle_posix_error('setpriority', c_setpriority(which, who, prio))
 
+    c_sched_get_priority_max = external('sched_get_priority_max', [rffi.INT],
+                              rffi.INT, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_sched_get_priority_min = external('sched_get_priority_min', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @enforceargs(int)
+    def sched_get_priority_max(policy):
+        return handle_posix_error('sched_get_priority_max', c_sched_get_priority_max(policy))
+
+    @enforceargs(int)
+    def sched_get_priority_min(policy):
+        return handle_posix_error('sched_get_priority_min', c_sched_get_priority_min(policy))
+
+
+
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -781,3 +781,27 @@
             raise
     finally:
         os.close(fd)
+
+ at rposix_requires('sched_get_priority_max')
+def test_sched_get_priority_max():
+    assert rposix.sched_get_priority_max(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_sched_get_priority_min():
+    assert rposix.sched_get_priority_min(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_os_sched_priority_max_greater_than_min():
+    policy = rposix.SCHED_RR
+    low = rposix.sched_get_priority_min(policy)
+    high = rposix.sched_get_priority_max(policy)
+    assert isinstance(low, int) == True
+    assert isinstance(high, int) == True
+    assert  high > low
+

From pypy.commits at gmail.com  Tue May 30 07:37:44 2017
From: pypy.commits at gmail.com (nanjekye)
Date: Tue, 30 May 2017 04:37:44 -0700 (PDT)
Subject: [pypy-commit] pypy jumbojet: sched_get min/max
Message-ID: <592d5988.84ae1c0a.e23f9.d174@mx.google.com>

Author: Joannah Nanjekye <nanjekyejoannah at gmail.com>
Branch: jumbojet
Changeset: r91443:d70723533eef
Date: 2017-04-08 21:35 +0300
http://bitbucket.org/pypy/pypy/changeset/d70723533eef/

Log:	sched_get min/max

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -229,6 +229,14 @@
         'POSIX_FADV_RANDOM', 'POSIX_FADV_NOREUSE', 'POSIX_FADV_DONTNEED']:
             assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+    
+    if hasattr(rposix, 'sched_get_priority_max'):
+        interpleveldefs['sched_get_priority_max'] = 'interp_posix.sched_get_priority_max'
+        interpleveldefs['sched_get_priority_min'] = 'interp_posix.sched_get_priority_min'
+        for _name in ['SCHED_FIFO', 'SCHED_RR', 'SCHED_OTHER',
+        'SCHED_BATCH']:
+            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
+            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
 
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2418,3 +2418,31 @@
             except OSError as e:
                 wrap_oserror(space, e, eintr_retry=True)
     return space.newint(res)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_max(space, policy):
+    """returns the maximum priority value that 
+    can be used with the scheduling algorithm 
+    identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_max(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_min(space, policy):
+    """rreturns the minimum priority value that
+     can be used with the scheduling algorithm 
+     identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_min(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
\ No newline at end of file
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -938,6 +938,34 @@
             assert os.WIFEXITED(status1)
             assert os.WEXITSTATUS(status1) == 0   # else, test failure
 
+    if hasattr(rposix, 'sched_get_priority_max'):
+        def test_os_sched_get_priority_max(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_max(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_BATCH) != -1
+
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_get_priority_min(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_min(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_BATCH) != -1
+            
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_priority_max_greater_than_min(self):
+            posix, os = self.posix, self.os
+            policy = posix.SCHED_RR
+            low = posix.sched_get_priority_min(policy)
+            high = posix.sched_get_priority_max(policy)
+            assert isinstance(low, int) == True
+            assert isinstance(high, int) == True
+            assert  high > low
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -235,6 +235,7 @@
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'sys/resource.h',
+                'sched.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
@@ -261,6 +262,10 @@
     POSIX_FADV_RANDOM= rffi_platform.DefinedConstantInteger('POSIX_FADV_RANDOM')
     POSIX_FADV_NOREUSE = rffi_platform.DefinedConstantInteger('POSIX_FADV_NOREUSE')
     POSIX_FADV_DONTNEED = rffi_platform.DefinedConstantInteger('POSIX_FADV_DONTNEED')
+    SCHED_FIFO = rffi_platform.DefinedConstantInteger('SCHED_FIFO')
+    SCHED_RR = rffi_platform.DefinedConstantInteger('SCHED_RR')
+    SCHED_OTHER = rffi_platform.DefinedConstantInteger('SCHED_OTHER')
+    SCHED_BATCH = rffi_platform.DefinedConstantInteger('SCHED_BATCH')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -1804,6 +1809,21 @@
     def setpriority(which, who, prio):
         handle_posix_error('setpriority', c_setpriority(which, who, prio))
 
+    c_sched_get_priority_max = external('sched_get_priority_max', [rffi.INT],
+                              rffi.INT, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_sched_get_priority_min = external('sched_get_priority_min', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @enforceargs(int)
+    def sched_get_priority_max(policy):
+        return handle_posix_error('sched_get_priority_max', c_sched_get_priority_max(policy))
+
+    @enforceargs(int)
+    def sched_get_priority_min(policy):
+        return handle_posix_error('sched_get_priority_min', c_sched_get_priority_min(policy))
+
+
+
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -785,3 +785,27 @@
             raise
     finally:
         os.close(fd)
+
+ at rposix_requires('sched_get_priority_max')
+def test_sched_get_priority_max():
+    assert rposix.sched_get_priority_max(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_sched_get_priority_min():
+    assert rposix.sched_get_priority_min(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_os_sched_priority_max_greater_than_min():
+    policy = rposix.SCHED_RR
+    low = rposix.sched_get_priority_min(policy)
+    high = rposix.sched_get_priority_max(policy)
+    assert isinstance(low, int) == True
+    assert isinstance(high, int) == True
+    assert  high > low
+

From pypy.commits at gmail.com  Tue May 30 07:37:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 04:37:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <592d598e.b885df0a.cad2b.bc3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91446:789fb549e0af
Date: 2017-05-30 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/789fb549e0af/

Log:	merge heads

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -357,6 +357,26 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+C-API Differences
+-----------------
+
+The external C-API has been reimplemented in PyPy as an internal cpyext module.
+We support most of the documented C-API, but sometimes internal C-abstractions
+leak out on CPython and are abused, perhaps even unknowingly. For instance,
+assignment to a ``PyTupleObject`` is not supported after the tuple is
+used internally, even by another C-API function call. On CPython this will
+succeed as long as the refcount is 1.  On PyPy this will always raise a
+``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
+exception (explicitly listed here for search engines).
+
+Another similar problem is assignment of a new function pointer to any of the
+``tp_as_*`` structures after calling ``PyType_Ready``. For instance, overriding
+``tp_as_number.nb_int`` with a different function after calling ``PyType_Ready``
+on CPython will result in the old function being called for ``x.__int__()``
+(via class ``__dict__`` lookup) and the new function being called for ``int(x)``
+(via slot lookup). On PyPy we will always call the __new__ function, not the
+old, this quirky behaviour is unfortunately necessary to fully support NumPy.
+
 Performance Differences
 -------------------------
 
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -161,3 +161,47 @@
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
+
+    def test_setitem(self):
+        module = self.import_extension('foo', [
+            ("set_after_use", "METH_O",
+             """
+                PyObject *t2, *tuple = PyTuple_New(1);
+                PyObject * one = PyLong_FromLong(1);
+                int res;
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                Py_INCREF(args);
+                res = PyTuple_SetItem(tuple, 0, args);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                /* Do something that uses the tuple, but does not incref */
+                t2 = PyTuple_GetSlice(tuple, 0, 1);
+                Py_DECREF(t2);
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                Py_DECREF(tuple);
+                if (res != 0)
+                {
+                    Py_DECREF(one);
+                    return NULL;
+                }
+                Py_INCREF(Py_None);
+                return Py_None;
+             """),
+            ])
+        import sys
+        s = 'abc'
+        if '__pypy__' in sys.builtin_module_names:
+            raises(SystemError, module.set_after_use, s)
+        else:
+            module.set_after_use(s)
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -6,7 +6,7 @@
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
-    track_reference, make_typedescr, get_typedescr)
+    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -132,19 +132,20 @@
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyTuple_SetItem(space, ref, index, py_obj):
-    # XXX this will not complain when changing tuples that have
-    # already been realized as a W_TupleObject, but won't update the
-    # W_TupleObject
     if not tuple_check_ref(space, ref):
         decref(space, py_obj)
         PyErr_BadInternalCall(space)
-    ref = rffi.cast(PyTupleObject, ref)
-    size = ref.c_ob_size
+    tupleobj = rffi.cast(PyTupleObject, ref)
+    size = tupleobj.c_ob_size
     if index < 0 or index >= size:
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
-    old_ref = ref.c_ob_item[index]
-    ref.c_ob_item[index] = py_obj    # consumes a reference
+    old_ref = tupleobj.c_ob_item[index]
+    if pyobj_has_w_obj(ref):
+        # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
+        raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
+                                        " use of tuple")
+    tupleobj.c_ob_item[index] = py_obj    # consumes a reference
     if old_ref:
         decref(space, old_ref)
     return 0

From pypy.commits at gmail.com  Tue May 30 07:47:38 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 30 May 2017 04:47:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: typo
Message-ID: <592d5bda.d4a9df0a.3454e.a8ed@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r91447:052ba96573f4
Date: 2017-05-30 13:47 +0200
http://bitbucket.org/pypy/pypy/changeset/052ba96573f4/

Log:	typo

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2435,7 +2435,7 @@
 
 @unwrap_spec(policy=int)
 def sched_get_priority_min(space, policy):
-    """rreturns the minimum priority value that
+    """returns the minimum priority value that
      can be used with the scheduling algorithm 
      identified by policy
     """
@@ -2445,4 +2445,4 @@
         except OSError as e:
             wrap_oserror(space, e, eintr_retry=True)
         else:
-           return space.newint(s)
\ No newline at end of file
+           return space.newint(s)

From pypy.commits at gmail.com  Tue May 30 07:52:03 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 30 May 2017 04:52:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: some missing docstrings
Message-ID: <592d5ce3.0882df0a.fc300.8cd3@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r91448:b28959100775
Date: 2017-05-30 13:51 +0200
http://bitbucket.org/pypy/pypy/changeset/b28959100775/

Log:	some missing docstrings

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2376,6 +2376,10 @@
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
+    """get_blocking(fd) -> bool
+
+Get the blocking mode of the file descriptor:
+False if the O_NONBLOCK flag is set, True if the flag is cleared."""
     try:
         flags = rposix.get_status_flags(fd)
     except OSError as e:
@@ -2384,6 +2388,12 @@
 
 @unwrap_spec(fd=c_int, blocking=int)
 def set_blocking(space, fd, blocking):
+    """\
+set_blocking(fd, blocking)
+
+Set the blocking mode of the specified file descriptor.
+Set the O_NONBLOCK flag if blocking is False,
+clear the O_NONBLOCK flag otherwise."""
     try:
         flags = rposix.get_status_flags(fd)
         if blocking:
@@ -2396,6 +2406,10 @@
 
 @unwrap_spec(out=c_int, count=int)
 def sendfile(space, out, w_in, w_offset, count):
+    """\
+sendfile(out, in, offset, count[, headers][, trailers], flags=0)
+            -> byteswritten
+Copy count bytes from file descriptor in to file descriptor out."""
     # why is an argument called "in"???  that doesn't make sense (it is
     # a reserved word), but that's what CPython does
     in_ = space.c_int_w(w_in)

From pypy.commits at gmail.com  Tue May 30 09:53:13 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:13 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: update version to 5.8.0
 final
Message-ID: <592d7949.658edf0a.3f7e1.4887@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91450:10736d300498
Date: 2017-05-30 16:33 +0300
http://bitbucket.org/pypy/pypy/changeset/10736d300498/

Log:	update version to 5.8.0 final

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.13"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.7.1"
-#define PYPY_VERSION_NUM  0x05070100
+#define PYPY_VERSION "5.8.0"
+#define PYPY_VERSION_NUM  0x05080000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 7, 1, "final", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 8, 0, "final", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue May 30 09:53:11 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:11 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release
Message-ID: <592d7947.83361c0a.a93ad.ddb0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91449:34e9d2c7a2e1
Date: 2017-05-30 16:14 +0300
http://bitbucket.org/pypy/pypy/changeset/34e9d2c7a2e1/

Log:	merge default into release

diff too long, truncating to 2000 out of 52647 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -49,6 +49,11 @@
 ^rpython/translator/goal/target.+-c$
 ^rpython/translator/goal/.+\.exe$
 ^rpython/translator/goal/.+\.dll$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.guess$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$
 ^pypy/goal/pypy-translation-snapshot$
 ^pypy/goal/pypy-c
 ^pypy/goal/.+\.exe$
@@ -80,5 +85,8 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
+^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
+
+
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,3 +34,7 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
+fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
+b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
+1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
+2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -61,12 +61,12 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
-    g['CC'] = "gcc -pthread"
-    g['CXX'] = "g++ -pthread"
+    g['CC'] = "cc -pthread"
+    g['CXX'] = "c++ -pthread"
     g['OPT'] = "-DNDEBUG -O2"
     g['CFLAGS'] = "-DNDEBUG -O2"
     g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "gcc -pthread -shared"
+    g['LDSHARED'] = "cc -pthread -shared"
     g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -36,9 +36,9 @@
 except ImportError:
     def _delitem_if_value_is(d, key, value):
         try:
-            if self.data[key] is value:  # fall-back: there is a potential
+            if d[key] is value:  # fall-back: there is a potential
                 #             race condition in multithreaded programs HERE
-                del self.data[key]
+                del d[key]
         except KeyError:
             pass
 
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -85,8 +85,9 @@
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
-        res._base = base
-        res._index = index
+        if base is not None:
+            res._base = base
+            res._index = index
         return res
 
     def _CData_retval(self, resbuffer):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -64,8 +64,9 @@
         res = object.__new__(self)
         res.__class__ = self
         res.__dict__['_buffer'] = resbuffer
-        res.__dict__['_base'] = base
-        res.__dict__['_index'] = index
+        if base is not None:
+            res.__dict__['_base'] = base
+            res.__dict__['_index'] = index
         return res
 
     def _CData_retval(self, resbuffer):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,9 +26,9 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
-WIN64 = sys.platform == 'win32' and sys.maxint == 2**63 - 1
+WIN64 = sys.platform == 'win32' and sys.maxint == 2 ** 63 - 1
 
 
 def get_com_error(errcode, riid, pIunk):
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -94,14 +97,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -126,27 +124,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -156,7 +153,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -168,15 +165,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -188,7 +188,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -201,7 +201,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -222,10 +222,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -259,7 +257,7 @@
         if (sys.platform == 'win32' and isinstance(argument, (int, long))
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -281,6 +279,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -317,7 +316,7 @@
             except:
                 exc_info = sys.exc_info()
                 traceback.print_tb(exc_info[2], file=sys.stderr)
-                print >>sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
+                print >> sys.stderr, "%s: %s" % (exc_info[0].__name__, exc_info[1])
                 return 0
             if self._restype_ is not None:
                 return res
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -234,6 +234,9 @@
         if ('_abstract_' in cls.__dict__ or cls is Structure 
                                          or cls is union.Union):
             raise TypeError("abstract class")
+        if hasattr(cls, '_swappedbytes_'):
+            raise NotImplementedError("missing in PyPy: structure/union with "
+                                      "swapped (non-native) byte ordering")
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -79,10 +79,20 @@
 BOOL WINAPI CreateProcessA(char *, char *, void *,
                            void *, BOOL, DWORD, char *,
                            char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
+BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
+                           void *, BOOL, DWORD, wchar_t *,
+                           wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 DWORD WINAPI WaitForSingleObject(HANDLE, DWORD);
 BOOL WINAPI GetExitCodeProcess(HANDLE, LPDWORD);
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
+DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS     0x0001
+#define SEM_NOGPFAULTERRORBOX      0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX     0x8000
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x50\x03\x00\x00\x13\x11\x00\x00\x53\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x4F\x03\x00\x00\x4E\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x42\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x00\x0F\x00\x00\x42\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x52\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x4C\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x49\x23GetStdHandle',0,b'\x00\x00\x3F\x23GetVersion',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x44\x23_getwch',0,b'\x00\x00\x44\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x46\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x41\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x4E\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x4F\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x42\x11wShowWindow',b'\x00\x00\x42\x11cbReserved2',b'\x00\x00\x51\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x4EPROCESS_INFORMATION',b'\x00\x00\x00\x4FSTARTUPINFO',b'\x00\x00\x00\x42wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
 )
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -31,10 +31,11 @@
 import weakref
 from threading import _get_ident as _thread_get_ident
 try:
-    from __pypy__ import newlist_hint
+    from __pypy__ import newlist_hint, add_memory_pressure
 except ImportError:
     assert '__pypy__' not in sys.builtin_module_names
     newlist_hint = lambda sizehint: []
+    add_memory_pressure = lambda size: None
 
 if sys.version_info[0] >= 3:
     StandardError = Exception
@@ -150,6 +151,9 @@
 def connect(database, timeout=5.0, detect_types=0, isolation_level="",
                  check_same_thread=True, factory=None, cached_statements=100):
     factory = Connection if not factory else factory
+    # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
+    # backed by :memory: or a file)
+    add_memory_pressure(100 * 1024)
     return factory(database, timeout, detect_types, isolation_level,
                     check_same_thread, factory, cached_statements)
 
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -42,8 +42,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
@@ -219,9 +220,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -79,6 +79,9 @@
 _ssl
     libssl
 
+_vmprof
+    libunwind (optional, loaded dynamically at runtime)
+
 Make sure to have these libraries (with development headers) installed
 before building PyPy, otherwise the resulting binary will not contain
 these modules.  Furthermore, the following libraries should be present
@@ -107,22 +110,22 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev \
+    tk-dev libgc-dev python-cffi \
     liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel \
+    gdbm-devel python-cffi\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses \
-    xz-devel  # For lzma on PyPy3.
+    libexpat-devel libffi-devel python-curses python-cffi \
+    xz-devel # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
 On Mac OS X, most of these build-time dependencies are installed alongside
@@ -185,7 +188,7 @@
 ::
 
     cd pypy/tool/release
-    ./package.py pypy-VER-PLATFORM
+    ./package.py --archive-name=pypy-VER-PLATFORM
 
 This creates a clean and prepared hierarchy, as well as a ``.tar.bz2``
 with the same content; both are found by default in
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -357,6 +357,24 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+Performance Differences
+-------------------------
+
+CPython has an optimization that can make repeated string concatenation not
+quadratic. For example, this kind of code runs in O(n) time::
+
+    s = ''
+    for string in mylist:
+        s += string
+
+In PyPy, this code will always have quadratic complexity. Note also, that the
+CPython optimization is brittle and can break by having slight variations in
+your code anyway. So you should anyway replace the code with::
+
+    parts = []
+    for string in mylist:
+        parts.append(string)
+    s = "".join(parts)
 
 Miscellaneous
 -------------
@@ -483,7 +501,14 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.7.1.rst
    release-v5.7.0.rst
    release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
@@ -59,6 +60,7 @@
 
 .. toctree::
 
+   release-v5.7.1.rst
    release-v5.7.0.rst
 
 CPython 3.3 compatible versions
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -32,10 +32,10 @@
 
 .. code-block:: console
 
-    $ tar xf pypy-2.1.tar.bz2
-    $ ./pypy-2.1/bin/pypy
-    Python 2.7.3 (480845e6b1dd, Jul 31 2013, 11:05:31)
-    [PyPy 2.1.0 with GCC 4.4.3] on linux2
+    $ tar xf pypy-x.y.z.tar.bz2
+    $ ./pypy-x.y.z/bin/pypy
+    Python 2.7.x (xxxxxxxxxxxx, Date, Time)
+    [PyPy x.y.z with GCC x.y.z] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     And now for something completely different: ``PyPy is an exciting technology
     that lets you to write fast, portable, multi-platform interpreters with less
@@ -57,6 +57,7 @@
 .. code-block:: console
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install -U pip wheel # to upgrade to the latest versions
     $ ./pypy-xxx/bin/pip install pygments  # for example
 
 Third party libraries will be installed in ``pypy-xxx/site-packages``, and
@@ -77,7 +78,17 @@
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
 
-Note that bin/python is now a symlink to bin/pypy.
+	# in any case activate it
+	$ source my-pypy-env/bin/activate
+
+Note that my-pypy-env/bin/python is now a symlink to my-pypy-env/bin/pypy
+so you should be able to run pypy simply by typing::
+
+    $ python
+
+You should still upgrade pip and wheel to the latest versions via::
+
+    $ my-pypy-env/bin/pip install -U pip wheel
 
 .. _pip: http://pypi.python.org/pypi/pip
 .. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -2,8 +2,11 @@
 PyPy2.7 and PyPy3.5 v5.7 - two in one release
 =============================================
 
-We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for
-Linux 64bit only at first).
+The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
 3.5.3.
@@ -88,14 +91,15 @@
 * New features and cleanups
 
   * update the format of the PYPYLOG file and improvements to vmprof
-  * emit more sysconfig values for downstream cextension packages
+  * emit more sysconfig values for downstream cextension packages including
+    properly setting purelib and platlib to site-packages
   * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``,
     ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``,
   * detect and raise on recreation of a PyPy object from a PyObject during
     tp_dealloc
   * refactor and clean up poor handling of unicode exposed in work on py3.5
   * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed)
-  * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14
+  * adapt ``weakref`` according to CPython issue 19542_, will be in CPython 2.7.14
   * support translations with cpyext and the Boehm GC (for special cases like
     RevDB_
   * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is
@@ -121,16 +125,18 @@
   * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only
     added in 10.12
   * support ``HAVE_FSTATVFS`` which was unintentionally always false
-  * fix user-created C-API heaptype, issue #2434_
+  * fix user-created C-API heaptype, issue 2434_
   * fix ``PyDict_Update`` is not actually the same as ``dict.update``
   * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute
-    issue #2446_
+    issue 2446_
   * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``,
     ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release``
   * improve support for creating C-extension objects from app-level classes,
     filling more slots, especially ``tp_new`` and ``tp_dealloc``
-  * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_
+  * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue 2475_
   * fix in corner cases with the GIL and C-API functions
+  * allow overriding thread.local.__init__ in a subclass, issue 2501_
+  * allow ``PyClass_New`` to be called with NULL as the first arguemnt, issue 2504_
 
 
 * Performance improvements:
@@ -183,21 +189,19 @@
 * Performance improvements:
 
   * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called
-  * 
-  * 
-  * 
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
   * PEP 489: Multi-phase extension module initialization
-  * XXX what else?
 
 .. _resolved: whatsnew-pypy2-5.7.0.html
 .. _19542: https://bugs.python.org/issue19542
 .. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys
 .. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on
 .. _2475: https://bitbucket.org/pypy/pypy/issues/2475
+.. _2501: https://bitbucket.org/pypy/pypy/issues/2501
+.. _2504: https://bitbucket.org/pypy/pypy/issues/2504
 .. _RevDB: https://bitbucket.org/pypy/revdb
 .. _cryptography: https://cryptography.io
 .. _cppyy: cppyy.html
diff --git a/pypy/doc/release-v5.7.1.rst b/pypy/doc/release-v5.7.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.7.1.rst
@@ -0,0 +1,50 @@
+==========
+PyPy 5.7.1
+==========
+
+We have released a bugfix PyPy2.7-v5.7.1 and PyPy3.5-v5.7.1 beta (Linux 64bit),
+due to the following issues:
+
+  * correctly handle an edge case in dict.pop (issue 2508_)
+
+  * fix a regression to correctly handle multiple inheritance in a C-API type
+    where the second base is an app-level class with a ``__new__`` function
+
+  * fix a regression to fill a C-API type's ``tp_getattr`` slot from a
+    ``__getattr__`` method (issue 2523_)
+
+Thanks to those who reported the issues.
+
+.. _2508: https://bitbucket.org/pypy/pypy/issues/2508
+.. _2523: https://bitbucket.org/pypy/pypy/issues/2523
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -101,6 +101,8 @@
     assert not not_documented
     if branch == 'default':
         assert not not_merged
+    else:
+        assert branch in documented, 'Please document this branch before merging: %s' % branch
 
 def test_startrev_on_default():
     doc = ROOT.join('pypy', 'doc')
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,7 +2,69 @@
 What's new in PyPy2.7 5.8+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7
+.. this is a revision shortly after release-pypy2.7-v5.7.0
 .. startrev: 44f31f6dd39f
 
+Add cpyext interfaces for ``PyModule_New``
 
+Correctly handle `dict.pop`` where the ``pop``
+key is not the same type as the ``dict``'s and ``pop``
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+.. branch: lstrip_to_empty_string
+
+.. branch: vmprof-native
+
+PyPy support to profile native frames in vmprof.
+
+.. branch: reusing-r11
+.. branch: branch-prediction
+
+Performance tweaks in the x86 JIT-generated machine code: rarely taken
+blocks are moved off-line.  Also, the temporary register used to contain
+large constants is reused across instructions.
+
+.. branch: vmprof-0.4.4
+
+.. branch: controller-refactor
+
+Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -23,11 +23,11 @@
 Installing Visual Compiler v9 (for Python 2.7)
 ----------------------------------------------
 
-This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
 was checked in Nov 2016). Note that the compiler suite will be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
-Using a current version of ``setuptools`` will be able to find it there. For
+A current version of ``setuptools`` will be able to find it there. For
 Windows 10, you must right-click the download, and under ``Properties`` ->
 ``Compatibility`` mark it as ``Run run this program in comatibility mode for``
 ``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
@@ -40,8 +40,9 @@
 Translating PyPy with Visual Studio
 -----------------------------------
 
-We routinely test translation using Visual Studio 2008, Express
-Edition.  Other configurations may work as well.
+We routinely test translation using v9, also known as Visual Studio 2008.
+Our buildbot is still using the Express Edition, not the compiler noted above.
+Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
 for the compiler, so you do not need to run vcvars before translation.
@@ -117,6 +118,9 @@
 -----------------------------------------------------------
 
 Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
+(for post-5.7.1 builds) with sha256 checksum 
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -124,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -173,13 +177,14 @@
 The zlib compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Download http://www.gzip.org/zlib/zlib-1.2.3.tar.gz and extract it in
-the base directory.  Then compile as a static library::
+Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in
+the base directory.  Then compile::
 
-    cd zlib-1.2.3
+    cd zlib-1.2.11
     nmake -f win32\Makefile.msc
     copy zlib.lib <somewhere in LIB>
     copy zlib.h zconf.h <somewhere in INCLUDE>
+    copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes)
 
 
 The bz2 compression library
@@ -198,22 +203,23 @@
 
 PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi
 wrapper is compiled when the module is imported for the first time.
-The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility.
+The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility.
 
 
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
 Download the source code of expat on sourceforge:
-http://sourceforge.net/projects/expat/ and extract it in the base directory.
-Version 2.1.0 is known to pass tests. Then open the project file ``expat.dsw``
+https://github.com/libexpat/libexpat/releases and extract it in the base directory.
+Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
 with Visual Studio; follow the instruction for converting the project files,
 switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build.
+reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for
+the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes)
 
-Then, copy the file ``win32\bin\release\libexpat.lib`` somewhere in somewhere
-in LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` somewhere in
-INCLUDE.
+Then, copy the file ``win32\bin\release\libexpat.lib`` into
+LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in
+INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH.
 
 
 The OpenSSL library
@@ -222,16 +228,17 @@
 OpenSSL needs a Perl interpreter to configure its makefile.  You may
 use the one distributed by ActiveState, or the one from cygwin.::
 
-    svn export http://svn.python.org/projects/external/openssl-1.0.1i
-    cd openssl-1.0.1i
+    svn export http://svn.python.org/projects/external/openssl-1.0.2k
+    cd openssl-1.0.2k
     perl Configure VC-WIN32 no-idea no-mdc2
     ms\do_ms.bat
     nmake -f ms\nt.mak install
+    copy out32\*.lib <somewhere in LIB>
+    xcopy /S include\openssl <somewhere in INCLUDE>
 
-Then, copy the files ``out32\*.lib`` somewhere in
-somewhere in LIB, and the entire ``include\openssl`` directory as-is somewhere
-in INCLUDE.
-
+For tests you will also need the dlls::
+    nmake -f ms\ntdll.mak install
+    copy out32dll\*.dll <somewhere in PATH>
 
 TkInter module support
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -241,18 +248,17 @@
 directory found for the release script, create the dlls, libs, headers and
 runtime by running::
 
-	svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
-	svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
-	cd tcl85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
-	nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
-	cd ..\..\tk85\win
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
-	nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
-
-Now you should have a tcktk\bin, tcltk\lib, and tcltk\include directory ready
-for use. The release packaging script will pick up the tcltk runtime in the lib
-directory and put it in the archive.
+    svn export http://svn.python.org/projects/external/tcl-8.5.2.1 tcl85
+    svn export http://svn.python.org/projects/external/tk-8.5.2.0 tk85
+    cd tcl85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=0 INSTALLDIR=..\..\tcltk clean all
+    nmake -f makefile.vc DEBUG=0 INSTALLDIR=..\..\tcltk install
+    cd ..\..\tk85\win
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 clean all
+    nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl85 install
+    copy ..\..\tcltk\bin\* <somewhere in PATH>
+    copy ..\..\tcltk\lib\*.lib <somewhere in LIB>
+    xcopy /S ..\..\tcltk\include <somewhere in INCLUDE>
 
 The lzma compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -263,6 +269,9 @@
 http://tukaani.org/xz/xz-5.0.5-windows.zip, check the signature
 http://tukaani.org/xz/xz-5.0.5-windows.zip.sig
 
+Then copy the headers to the include directory, rename ``liblzma.a`` to 
+``lzma.lib`` and copy it to the lib directory
+
 
 Using the mingw compiler
 ------------------------
@@ -334,9 +343,9 @@
 integer.  The simplest fix is to make sure that it is so, but it will
 give the following incompatibility between CPython and PyPy on Win64:
 
-CPython: ``sys.maxint == 2**32-1, sys.maxsize == 2**64-1``
+CPython: ``sys.maxint == 2**31-1, sys.maxsize == 2**63-1``
 
-PyPy: ``sys.maxint == sys.maxsize == 2**64-1``
+PyPy: ``sys.maxint == sys.maxsize == 2**63-1``
 
 ...and, correspondingly, PyPy supports ints up to the larger value of
 sys.maxint before they are converted to ``long``.  The first decision
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -264,11 +264,14 @@
                 raise Exception("Cannot use the --output option with PyPy "
                                 "when --shared is on (it is by default). "
                                 "See issue #1971.")
-            if (config.translation.profopt is not None
-                    and not config.translation.noprofopt):
-                raise Exception("Cannot use the --profopt option "
-                                "when --shared is on (it is by default). "
-                                "See issue #2398.")
+
+        # if both profopt and profoptpath are specified then we keep them as they are with no other changes
+        if config.translation.profopt:
+            if config.translation.profoptargs is None:
+                config.translation.profoptargs = "$(RPYDIR)/../lib-python/2.7/test/regrtest.py --pgo -x test_asyncore test_gdb test_multiprocessing test_subprocess || true"
+        elif config.translation.profoptargs is not None:
+            raise Exception("Cannot use --profoptargs without specifying --profopt as well")
+
         if sys.platform == 'win32':
             libdir = thisdir.join('..', '..', 'libs')
             libdir.ensure(dir=1)
@@ -307,6 +310,9 @@
             config.translation.jit = True
 
         if config.translation.sandbox:
+            assert 0, ("--sandbox is not tested nor maintained.  If you "
+                       "really want to try it anyway, remove this line in "
+                       "pypy/goal/targetpypystandalone.py.")
             config.objspace.lonepycfiles = False
 
         if config.objspace.usemodules.cpyext:
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -818,6 +818,11 @@
     executable = sys.pypy_find_executable(executable)
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
+        for lib_path in sys.path:
+            stdlib_path = sys.pypy_find_stdlib(lib_path)
+            if stdlib_path is not None:
+                break
+    if stdlib_path is None:
         print >> sys.stderr, STDLIB_WARNING
     else:
         sys.path[:] = stdlib_path
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -7,6 +7,7 @@
 from pypy.interpreter.astcompiler import ast, misc, symtable
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.pycode import PyCode
+from pypy.interpreter.miscutils import string_sort
 from pypy.tool import stdlib_opcode as ops
 
 
@@ -138,9 +139,12 @@
 
 
 def _make_index_dict_filter(syms, flag):
+    names = syms.keys()
+    string_sort(names)   # return cell vars in alphabetical order
     i = 0
     result = {}
-    for name, scope in syms.iteritems():
+    for name in names:
+        scope = syms[name]
         if scope == flag:
             result[name] = i
             i += 1
@@ -170,6 +174,7 @@
         self.var_names = _list_to_dict(scope.varnames)
         self.cell_vars = _make_index_dict_filter(scope.symbols,
                                                  symtable.SCOPE_CELL)
+        string_sort(scope.free_vars)    # return free vars in alphabetical order
         self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars))
         self.w_consts = space.newdict()
         self.argcount = 0
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -36,7 +36,7 @@
         self.roles = {}
         self.varnames = []
         self.children = []
-        self.free_vars = []
+        self.free_vars = []    # a bag of names: the order doesn't matter here
         self.temp_name_counter = 1
         self.has_exec = False
         self.has_free = False
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -9,7 +9,9 @@
 from rpython.rlib.signature import signature
 from rpython.rlib.rarithmetic import r_uint, SHRT_MIN, SHRT_MAX, \
     INT_MIN, INT_MAX, UINT_MAX, USHRT_MAX
+from rpython.rlib.buffer import StringBuffer
 
+from pypy.interpreter.buffer import BufferInterfaceNotFound
 from pypy.interpreter.executioncontext import (ExecutionContext, ActionFlag,
     make_finalizer_queue)
 from pypy.interpreter.error import OperationError, new_exception_class, oefmt
@@ -221,7 +223,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(flags))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.buffer_w(space, flags)
         raise BufferInterfaceNotFound
 
@@ -233,7 +236,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.readbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -245,7 +249,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.writebuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -254,7 +259,8 @@
         if w_impl is not None:
             w_result = space.get_and_call_function(w_impl, self,
                                         space.newint(space.BUF_FULL_RO))
-            if space.isinstance_w(w_result, space.w_buffer):
+            if (space.isinstance_w(w_result, space.w_buffer) or
+                    space.isinstance_w(w_result, space.w_memoryview)):
                 return w_result.charbuf_w(space)
         raise BufferInterfaceNotFound
 
@@ -392,9 +398,6 @@
 class DescrMismatch(Exception):
     pass
 
-class BufferInterfaceNotFound(Exception):
-    pass
-
 @specialize.memo()
 def wrappable_class_name(Class):
     try:
@@ -1501,18 +1504,28 @@
     def readbuf_w(self, w_obj):
         # Old buffer interface, returns a readonly buffer (PyObject_AsReadBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_SIMPLE).as_readbuf()
+        except OperationError:
+            self._getarg_error("convertible to a buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.readbuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a readable buffer object")
+            self._getarg_error("convertible to a buffer", w_obj)
 
     def writebuf_w(self, w_obj):
         # Old buffer interface, returns a writeable buffer (PyObject_AsWriteBuffer)
         try:
+            return w_obj.buffer_w(self, self.BUF_WRITABLE).as_writebuf()
+        except OperationError:
+            self._getarg_error("read-write buffer", w_obj)
+        except BufferInterfaceNotFound:
+            pass
+        try:
             return w_obj.writebuf_w(self)
         except BufferInterfaceNotFound:
-            raise oefmt(self.w_TypeError,
-                        "expected a writeable buffer object")
+            self._getarg_error("read-write buffer", w_obj)
 
     def charbuf_w(self, w_obj):
         # Old buffer interface, returns a character buffer (PyObject_AsCharBuffer)
@@ -1541,12 +1554,10 @@
             if self.isinstance_w(w_obj, self.w_unicode):
                 return self.str(w_obj).readbuf_w(self)
             try:
-                return w_obj.buffer_w(self, 0)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.readbuf_w(self)
-            except BufferInterfaceNotFound:
+                return self.readbuf_w(w_obj)
+            except OperationError as e:
+                if not e.match(self, self.w_TypeError):
+                    raise
                 self._getarg_error("string or buffer", w_obj)
         elif code == 's#':
             if self.isinstance_w(w_obj, self.w_bytes):
@@ -1558,16 +1569,7 @@
             except BufferInterfaceNotFound:
                 self._getarg_error("string or read-only buffer", w_obj)
         elif code == 'w*':
-            try:
-                return w_obj.buffer_w(self, self.BUF_WRITABLE)
-            except OperationError:
-                self._getarg_error("read-write buffer", w_obj)
-            except BufferInterfaceNotFound:
-                pass
-            try:
-                return w_obj.writebuf_w(self)
-            except BufferInterfaceNotFound:
-                self._getarg_error("read-write buffer", w_obj)
+            return self.writebuf_w(w_obj)
         elif code == 't#':
             try:
                 return w_obj.charbuf_w(self)
@@ -1654,6 +1656,23 @@
     def fsencode_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.fsencode_w(w_obj)
 
+    def byte_w(self, w_obj):
+        """
+        Convert an index-like object to an interp-level char
+
+        Used for app-level code like "bytearray(b'abc')[0] = 42".
+        """
+        if self.isinstance_w(w_obj, self.w_bytes):
+            string = self.bytes_w(w_obj)
+            if len(string) != 1:
+                raise oefmt(self.w_ValueError, "string must be of size 1")
+            return string[0]
+        value = self.getindex_w(w_obj, None)
+        if not 0 <= value < 256:
+            # this includes the OverflowError in case the long is too large
+            raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
+        return chr(value)
+
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
diff --git a/pypy/interpreter/buffer.py b/pypy/interpreter/buffer.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/buffer.py
@@ -0,0 +1,295 @@
+from rpython.rlib.buffer import StringBuffer, SubBuffer
+
+from pypy.interpreter.error import oefmt
+
+class BufferInterfaceNotFound(Exception):
+    pass
+
+
+class BufferView(object):
+    """Abstract base class for buffers."""
+    _attrs_ = ['readonly']
+    _immutable_ = True
+
+    def getlength(self):
+        """Returns the size in bytes (even if getitemsize() > 1)."""
+        raise NotImplementedError
+
+    def as_str(self):
+        "Returns an interp-level string with the whole content of the buffer."
+        return ''.join(self._copy_buffer())
+
+    def getbytes(self, start, size):
+        """Return `size` bytes starting at byte offset `start`.
+
+        This is a low-level operation, it is up to the caller to ensure that
+        the data requested actually correspond to items accessible from the
+        BufferView.
+        Note that `start` may be negative, e.g. if the buffer is reversed.
+        """
+        raise NotImplementedError
+
+    def setbytes(self, start, string):
+        raise NotImplementedError
+
+    def get_raw_address(self):
+        raise ValueError("no raw buffer")
+
+    def as_readbuf(self):
+        # Inefficient. May be overridden.
+        return StringBuffer(self.as_str())
+
+    def as_writebuf(self):
+        """Return a writable Buffer sharing the same data as `self`."""
+        raise BufferInterfaceNotFound
+
+    def getformat(self):
+        raise NotImplementedError
+
+    def getitemsize(self):
+        raise NotImplementedError
+
+    def getndim(self):
+        raise NotImplementedError
+
+    def getshape(self):
+        raise NotImplementedError
+
+    def getstrides(self):
+        raise NotImplementedError
+
+    def releasebuffer(self):
+        pass
+
+    def value_from_bytes(self, space, s):
+        from pypy.module.struct.formatiterator import UnpackFormatIterator
+        buf = StringBuffer(s)
+        fmtiter = UnpackFormatIterator(space, buf)
+        fmtiter.interpret(self.getformat())
+        return fmtiter.result_w[0]
+
+    def _copy_buffer(self):
+        if self.getndim() == 0:
+            itemsize = self.getitemsize()
+            return [self.getbytes(0, itemsize)]
+        data = []
+        self._copy_rec(0, data, 0)
+        return data
+
+    def _copy_rec(self, idim, data, off):
+        shapes = self.getshape()
+        shape = shapes[idim]
+        strides = self.getstrides()
+
+        if self.getndim() - 1 == idim:
+            self._copy_base(data, off)
+            return
+
+        for i in range(shape):
+            self._copy_rec(idim + 1, data, off)
+            off += strides[idim]
+
+    def _copy_base(self, data, off):
+        shapes = self.getshape()
+        step = shapes[0]
+        strides = self.getstrides()
+        itemsize = self.getitemsize()
+        bytesize = self.getlength()
+        copiedbytes = 0
+        for i in range(step):
+            bytes = self.getbytes(off, itemsize)
+            data.append(bytes)
+            copiedbytes += len(bytes)
+            off += strides[0]
+            # do notcopy data if the sub buffer is out of bounds
+            if copiedbytes >= bytesize:
+                break
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        shape = self.getshape()
+        nitems = shape[dim]
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        # TODO suboffsets?
+        strides = self.getstrides()
+        return strides[dim] * index
+
+    def w_getitem(self, space, idx):
+        offset = self.get_offset(space, 0, idx)
+        itemsize = self.getitemsize()
+        # TODO: this probably isn't very fast
+        data = self.getbytes(offset, itemsize)
+        return space.newbytes(data)
+
+    def new_slice(self, start, step, slicelength):
+        return BufferSlice(self, start, step, slicelength)
+
+    def w_tolist(self, space):
+        dim = self.getndim()
+        if dim == 0:
+            raise NotImplementedError
+        elif dim == 1:
+            n = self.getshape()[0]
+            values_w = [space.ord(self.w_getitem(space, i)) for i in range(n)]
+            return space.newlist(values_w)
+        else:
+            return self._tolist_rec(space, 0, 0)
+
+    def _tolist_rec(self, space, start, idim):
+        strides = self.getstrides()
+        shape = self.getshape()
+        #
+        dim = idim + 1
+        stride = strides[idim]
+        itemsize = self.getitemsize()
+        dimshape = shape[idim]
+        #
+        if dim >= self.getndim():
+            bytecount = (stride * dimshape)
+            values_w = [
+                self.value_from_bytes(space, self.getbytes(pos, itemsize))
+                for pos in range(start, start + bytecount, stride)]
+            return space.newlist(values_w)
+
+        items = [None] * dimshape
+        for i in range(dimshape):
+            item = self._tolist_rec(space, start, idim + 1)
+            items[i] = item
+            start += stride
+
+        return space.newlist(items)
+
+    def wrap(self, space):
+        return space.newmemoryview(self)
+
+
+class SimpleView(BufferView):
+    _attrs_ = ['readonly', 'data']
+    _immutable_ = True
+
+    def __init__(self, data):
+        self.data = data
+        self.readonly = self.data.readonly
+
+    def getlength(self):
+        return self.data.getlength()
+
+    def as_str(self):
+        return self.data.as_str()
+
+    def getbytes(self, start, size):
+        return self.data[start:start + size]
+
+    def setbytes(self, offset, s):
+        self.data.setslice(offset, s)
+
+    def get_raw_address(self):
+        return self.data.get_raw_address()
+
+    def as_readbuf(self):
+        return self.data
+
+    def as_writebuf(self):
+        assert not self.data.readonly
+        return self.data
+
+    def getformat(self):
+        return 'B'
+
+    def getitemsize(self):
+        return 1
+
+    def getndim(self):
+        return 1
+
+    def getshape(self):
+        return [self.getlength()]
+
+    def getstrides(self):
+        return [1]
+
+    def get_offset(self, space, dim, index):
+        "Convert index at dimension `dim` into a byte offset"
+        assert dim == 0
+        nitems = self.getlength()
+        if index < 0:
+            index += nitems
+        if index < 0 or index >= nitems:
+            raise oefmt(space.w_IndexError,
+                "index out of bounds on dimension %d", dim + 1)
+        return index
+
+    def w_getitem(self, space, idx):
+        idx = self.get_offset(space, 0, idx)
+        ch = self.data[idx]
+        return space.newbytes(ch)
+
+    def new_slice(self, start, step, slicelength):
+        if step == 1:
+            return SimpleView(SubBuffer(self.data, start, slicelength))
+        else:
+            return BufferSlice(self, start, step, slicelength)
+
+
+class BufferSlice(BufferView):
+    _immutable_ = True
+    _attrs_ = ['parent', 'readonly', 'shape', 'strides', 'start', 'step']
+
+    def __init__(self, parent, start, step, length):
+        self.parent = parent
+        self.readonly = self.parent.readonly
+        self.strides = parent.getstrides()[:]
+        self.start = start
+        self.step = step
+        self.strides[0] *= step
+        self.shape = parent.getshape()[:]
+        self.shape[0] = length
+
+    def getlength(self):
+        return self.shape[0] * self.getitemsize()
+
+    def getbytes(self, start, size):
+        offset = self.start * self.parent.getstrides()[0]
+        return self.parent.getbytes(offset + start, size)
+
+    def setbytes(self, start, string):
+        if len(string) == 0:
+            return        # otherwise, adding self.offset might make 'start'
+                          # out of bounds
+        offset = self.start * self.parent.getstrides()[0]
+        self.parent.setbytes(offset + start, string)
+
+    def get_raw_address(self):
+        from rpython.rtyper.lltypesystem import rffi
+        offset = self.start * self.parent.getstrides()[0]
+        return rffi.ptradd(self.parent.get_raw_address(), offset)
+
+    def getformat(self):
+        return self.parent.getformat()
+
+    def getitemsize(self):
+        return self.parent.getitemsize()
+
+    def getndim(self):
+        return self.parent.getndim()
+
+    def getshape(self):
+        return self.shape
+
+    def getstrides(self):
+        return self.strides
+
+    def parent_index(self, idx):
+        return self.start + self.step * idx
+
+    def w_getitem(self, space, idx):
+        return self.parent.w_getitem(space, self.parent_index(idx))
+
+    def new_slice(self, start, step, slicelength):
+        real_start = start + self.start
+        real_step = self.step * step
+        return BufferSlice(self.parent, real_start, real_step, slicelength)
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -2,6 +2,9 @@
 Miscellaneous utilities.
 """
 
+from rpython.rlib.listsort import make_timsort_class
+
+
 class ThreadLocals:
     """Pseudo thread-local storage, for 'space.threadlocals'.
     This is not really thread-local at all; the intention is that the PyPy
@@ -53,3 +56,15 @@
             def set(self, key, value):
                 self._dict[key] = value
         return FakeWeakValueDict()
+
+
+_StringBaseTimSort = make_timsort_class()
+
+class StringSort(_StringBaseTimSort):
+    def lt(self, a, b):
+        return a < b
+
+def string_sort(lst):
+    """Sort a (resizable) list of strings."""
+    sorter = StringSort(lst, len(lst))
+    sorter.sort()
diff --git a/pypy/interpreter/pyparser/future.py b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -78,6 +78,7 @@
     from pypy.interpreter.pyparser import pygram
     it = TokenIterator(tokens)
     result = 0
+    last_position = (0, 0)
     #
     # The only things that can precede a future statement are another
     # future statement and a doc string (only one).  This is a very
@@ -92,6 +93,11 @@
            it.skip_name("__future__") and
            it.skip_name("import")):
         it.skip(pygram.tokens.LPAR)    # optionally
+        # return in 'last_position' any line-column pair that points
+        # somewhere inside the last __future__ import statement
+        # (at the start would be fine too, but it's easier to grab a
+        # random position inside)
+        last_position = (it.tok[2], it.tok[3])
         result |= future_flags.get_compiler_feature(it.next_feature_name())
         while it.skip(pygram.tokens.COMMA):
             result |= future_flags.get_compiler_feature(it.next_feature_name())
@@ -99,5 +105,4 @@
         it.skip(pygram.tokens.SEMI)    # optionally
         it.skip_newlines()
 
-    position = (it.tok[2], it.tok[3])
-    return result, position
+    return result, last_position
diff --git a/pypy/interpreter/pyparser/test/test_future.py b/pypy/interpreter/pyparser/test/test_future.py
--- a/pypy/interpreter/pyparser/test/test_future.py
+++ b/pypy/interpreter/pyparser/test/test_future.py
@@ -2,10 +2,9 @@
 from pypy.interpreter.pyparser import future, pytokenizer
 from pypy.tool import stdlib___future__ as fut
 
-def run(s, expected_last_future=None):
+def run(s, expected_last_future=(0, 0)):
     source_lines = s.splitlines(True)
     tokens = pytokenizer.generate_tokens(source_lines, 0)
-    expected_last_future = expected_last_future or tokens[-1][2:4]
     #
     flags, last_future_import = future.add_future_flags(
         future.futureFlags_2_7, tokens)
@@ -14,7 +13,7 @@
 
 def test_docstring():
     s = '"Docstring\\" "\nfrom  __future__ import division\n'
-    f = run(s)
+    f = run(s, (2, 24))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_comment():
@@ -45,167 +44,167 @@
 
 def test_from():
     s = 'from  __future__ import division\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_froms():
     s = 'from  __future__ import division, generators, with_statement\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_from_as():
     s = 'from  __future__ import division as b\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == fut.CO_FUTURE_DIVISION
     
 def test_froms_as():
     s = 'from  __future__ import division as b, generators as c\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED)
 
 def test_from_paren():
     s = 'from  __future__ import (division)\n'
-    f = run(s)
+    f = run(s, (1, 25))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_froms_paren():
     s = 'from  __future__ import (division, generators)\n'
-    f = run(s)
+    f = run(s, (1, 25))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED)
 
 def test_froms_paren_as():
     s = 'from  __future__ import (division as b, generators,)\n'
-    f = run(s)
+    f = run(s, (1, 25))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED)
 
 def test_paren_with_newline():
     s = 'from __future__ import (division,\nabsolute_import)\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
 
 def test_paren_with_newline_2():
     s = 'from __future__ import (\ndivision,\nabsolute_import)\n'
-    f = run(s)
+    f = run(s, (2, 0))
     assert f == (fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_ABSOLUTE_IMPORT)
 
 def test_multiline():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,)\nfrom __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (4, 23))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_windows_style_lineendings():
     s = '"abc" #def\r\n  #ghi\r\nfrom  __future__ import (division as b, generators,)\r\nfrom __future__ import with_statement\r\n'
-    f = run(s)
+    f = run(s, (4, 23))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_mac_style_lineendings():
     s = '"abc" #def\r  #ghi\rfrom  __future__ import (division as b, generators,)\rfrom __future__ import with_statement\r'
-    f = run(s)
+    f = run(s, (4, 23))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_semicolon():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,);  from __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (3, 78))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_semicolon_2():
     s = 'from  __future__ import division; from foo import bar'
-    f = run(s, expected_last_future=(1, 39))
+    f = run(s, expected_last_future=(1, 24))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_full_chain():
     s = '"abc" #def\n  #ghi\nfrom  __future__ import (division as b, generators,);  from __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (3, 78))
     assert f == (fut.CO_FUTURE_DIVISION |
                  fut.CO_GENERATOR_ALLOWED |
                  fut.CO_FUTURE_WITH_STATEMENT)
 
 def test_intervening_code():
     s = 'from  __future__ import (division as b, generators,)\nfrom sys import modules\nfrom __future__ import with_statement\n'
-    f = run(s, expected_last_future=(2, 5))
+    f = run(s, expected_last_future=(1, 25))
     assert f == (fut.CO_FUTURE_DIVISION | fut.CO_GENERATOR_ALLOWED)
 
 def test_nonexisting():
     s = 'from  __future__ import non_existing_feature\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == 0
 
 def test_nonexisting_2():
     s = 'from  __future__ import non_existing_feature, with_statement\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_from_import_abs_import():
     s = 'from  __future__ import absolute_import\n'
-    f = run(s)
+    f = run(s, (1, 24))
     assert f == fut.CO_FUTURE_ABSOLUTE_IMPORT
 
 def test_raw_doc():
     s = 'r"Doc"\nfrom __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (2, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_unicode_doc():
     s = 'u"Doc"\nfrom __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (2, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_raw_unicode_doc():
     s = 'ru"Doc"\nfrom __future__ import with_statement\n'
-    f = run(s)
+    f = run(s, (2, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_continuation_line():
     s = "\\\nfrom __future__ import with_statement\n"
-    f = run(s)
+    f = run(s, (2, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_continuation_lines():
     s = "\\\n  \t\\\nfrom __future__ import with_statement\n"
-    f = run(s)
+    f = run(s, (3, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_lots_of_continuation_lines():
     s = "\\\n\\\n\\\n\\\n\\\n\\\n\nfrom __future__ import with_statement\n"
-    f = run(s)
+    f = run(s, (8, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT
 
 def test_continuation_lines_raise():
     s = "   \\\n  \t\\\nfrom __future__ import with_statement\n"
-    f = run(s, expected_last_future=(1, 0))
+    f = run(s)
     assert f == 0     # because of the INDENT
 
 def test_continuation_lines_in_docstring_single_quoted():
     s = '"\\\n\\\n\\\n\\\n\\\n\\\n"\nfrom  __future__ import division\n'
-    f = run(s)
+    f = run(s, (8, 24))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_continuation_lines_in_docstring_triple_quoted():
     s = '"""\\\n\\\n\\\n\\\n\\\n\\\n"""\nfrom  __future__ import division\n'
-    f = run(s)
+    f = run(s, (8, 24))
     assert f == fut.CO_FUTURE_DIVISION
 
 def test_blank_lines():
     s = ('\n\t\n\nfrom __future__ import with_statement'
          '  \n  \n  \nfrom __future__ import division')
-    f = run(s)
+    f = run(s, (7, 23))
     assert f == fut.CO_FUTURE_WITH_STATEMENT | fut.CO_FUTURE_DIVISION
 
 def test_dummy_semicolons():
     s = ('from __future__ import division;\n'
          'from __future__ import with_statement;')
-    f = run(s)
+    f = run(s, (2, 23))
     assert f == fut.CO_FUTURE_DIVISION | fut.CO_FUTURE_WITH_STATEMENT
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -787,6 +787,32 @@
         else:
             assert l1 == l2 == l3 == l4 == [1, 3, 2, 4]
 
+    def test_freevars_order(self):
+        # co_cellvars and co_freevars are guaranteed to appear in
+        # alphabetical order.  See CPython Issue #15368 (which does
+        # not come with tests).
+        source = """if 1:
+        def f1(x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15):
+            def g1():
+                return (x1,x2,x3,x4,x5,x6,x7,x8,x9,x10,x11,x12,x13,x14,x15)
+            return g1
+        def f2(x15,x14,x13,x12,x11,x10,x9,x8,x7,x6,x5,x4,x3,x2,x1):
+            def g2():
+                return (x15,x14,x13,x12,x11,x10,x9,x8,x7,x6,x5,x4,x3,x2,x1)
+            return g2
+        c1 = f1(*range(15)).__code__.co_freevars
+        c2 = f2(*range(15)).__code__.co_freevars
+        r1 = f1.__code__.co_cellvars
+        r2 = f2.__code__.co_cellvars
+        """
+        d = {}
+        exec(source, d)
+        assert d['c1'] == d['c2']
+        # the test above is important for a few bytecode hacks,
+        # but actually we get them in alphabetical order, so check that:
+        assert d['c1'] == tuple(sorted(d['c1']))
+        assert d['r1'] == d['r2'] == d['c1']
+
 
 ##class TestPythonAstCompiler(BaseTestCompiler):
 ##    def setup_method(self, method):
diff --git a/pypy/interpreter/test/test_syntax.py b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -366,6 +366,15 @@
         assert isinstance(ns["b"], str)
         assert isinstance(ns["c"], str)
 
+    def test_both_futures_with_semicolon(self):
+        # Issue #2526: a corner case which crashes only if the file
+        # contains *nothing more* than two __future__ imports separated
+        # by a semicolon.
+        s = """
+from __future__ import unicode_literals; from __future__ import print_function
+"""
+        exec s in {}
+
 
 class AppTestComprehensions:
 
diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -2,7 +2,6 @@
 Plain Python definition of the builtin functions oriented towards
 functional programming.
 """
-from __future__ import with_statement
 import operator
 from __pypy__ import resizelist_hint, newlist_hint
 from __pypy__ import specialized_zip_2_lists
diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -38,6 +38,8 @@
                     "compile() arg 3 must be 'exec', 'eval' or 'single'")
 
     if space.isinstance_w(w_source, space.gettypeobject(ast.W_AST.typedef)):
+        if flags & consts.PyCF_ONLY_AST:
+            return w_source
         ast_node = ast.mod.from_object(space, w_source)
         return ec.compiler.compile_ast(ast_node, filename, mode, flags)
 
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py

From pypy.commits at gmail.com  Tue May 30 09:53:19 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:19 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: update version to 5.8.0 beta
Message-ID: <592d794f.5b8bdf0a.2b76b.2cd2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91452:60728ba1c887
Date: 2017-05-30 16:36 +0300
http://bitbucket.org/pypy/pypy/changeset/60728ba1c887/

Log:	update version to 5.8.0 beta

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.5.3"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.7.1-beta0"
-#define PYPY_VERSION_NUM  0x05070100
+#define PYPY_VERSION "5.8.0-beta0"
+#define PYPY_VERSION_NUM  0x05080000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 7, 1, "beta", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 8, 0, "beta", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue May 30 09:53:17 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:17 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release
 branch
Message-ID: <592d794d.05a5df0a.4018.d677@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91451:0736d37d56a7
Date: 2017-05-30 16:35 +0300
http://bitbucket.org/pypy/pypy/changeset/0736d37d56a7/

Log:	merge py3.5 into release branch

diff too long, truncating to 2000 out of 57131 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -51,11 +51,18 @@
 ^rpython/translator/goal/target.+-c$
 ^rpython/translator/goal/.+\.exe$
 ^rpython/translator/goal/.+\.dll$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/Makefile$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.guess$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$
+^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$
 ^pypy/goal/pypy-translation-snapshot$
 ^pypy/goal/pypy-c
+^pypy/goal/pypy3-c
 ^pypy/goal/.+\.exe$
 ^pypy/goal/.+\.dll$
 ^pypy/goal/.+\.lib$
+^pypy/goal/.+\.dylib$
 ^pypy/_cache$
 ^lib-python/2.7/lib2to3/.+\.pickle$
 ^lib_pypy/__pycache__$
@@ -83,5 +90,8 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
+^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
+
+
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -34,4 +34,7 @@
 050d84dd78997f021acf0e133934275d63547cc0 release-pypy2.7-v5.4.1
 0e2d9a73f5a1818d0245d75daccdbe21b2d5c3ef release-pypy2.7-v5.4.1
 aff251e543859ce4508159dd9f1a82a2f553de00 release-pypy2.7-v5.6.0
-ea9979b550eeae87924dc4bef06070e8f8d0e22f release-pypy3.3-5.5.0
+fa3249d55d15b9829e1be69cdf45b5a44cec902d release-pypy2.7-v5.7.0
+b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
+1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
+2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
diff --git a/include/README b/include/README
--- a/include/README
+++ b/include/README
@@ -1,7 +1,11 @@
 This directory contains all the include files needed to build cpython
 extensions with PyPy.  Note that these are just copies of the original headers
-that are in pypy/module/cpyext/include: they are automatically copied from
-there during translation.
+that are in pypy/module/cpyext/{include,parse}: they are automatically copied
+from there during translation.
 
-Moreover, pypy_decl.h and pypy_macros.h are automatically generated, also
-during translation.
+Moreover, some pypy-specific files are automatically generated, also during
+translation. Currently they are:
+* pypy_decl.h
+* pypy_macros.h
+* pypy_numpy.h
+* pypy_structmember_decl.h
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -61,12 +61,12 @@
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
     g = {}
-    g['CC'] = "gcc -pthread"
-    g['CXX'] = "g++ -pthread"
+    g['CC'] = "cc -pthread"
+    g['CXX'] = "c++ -pthread"
     g['OPT'] = "-DNDEBUG -O2"
     g['CFLAGS'] = "-DNDEBUG -O2"
     g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "gcc -pthread -shared"
+    g['LDSHARED'] = "cc -pthread -shared"
     g['SO'] = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"
diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py
--- a/lib-python/2.7/sysconfig.py
+++ b/lib-python/2.7/sysconfig.py
@@ -29,8 +29,8 @@
     'pypy': {
         'stdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
         'platstdlib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'purelib': '{base}/lib-{implementation_lower}/{py_version_short}',
-        'platlib': '{base}/lib-{implementation_lower}/{py_version_short}',
+        'purelib': '{base}/site-packages',
+        'platlib': '{base}/site-packages',
         'include': '{base}/include',
         'platinclude': '{base}/include',
         'scripts': '{base}/bin',
diff --git a/lib-python/2.7/weakref.py b/lib-python/2.7/weakref.py
--- a/lib-python/2.7/weakref.py
+++ b/lib-python/2.7/weakref.py
@@ -36,9 +36,9 @@
 except ImportError:
     def _delitem_if_value_is(d, key, value):
         try:
-            if self.data[key] is value:  # fall-back: there is a potential
+            if d[key] is value:  # fall-back: there is a potential
                 #             race condition in multithreaded programs HERE
-                del self.data[key]
+                del d[key]
         except KeyError:
             pass
 
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -746,15 +746,15 @@
             _MONTHNAMES[self._month],
             self._day, self._year)
 
-    def strftime(self, fmt):
+    def strftime(self, format):
         "Format using strftime()."
-        return _wrap_strftime(self, fmt, self.timetuple())
+        return _wrap_strftime(self, format, self.timetuple())
 
-    def __format__(self, fmt):
-        if not isinstance(fmt, str):
-            raise TypeError("must be str, not %s" % type(fmt).__name__)
-        if len(fmt) != 0:
-            return self.strftime(fmt)
+    def __format__(self, format):
+        if not isinstance(format, str):
+            raise TypeError("must be str, not %s" % type(format).__name__)
+        if len(format) != 0:
+            return self.strftime(format)
         return str(self)
 
     def isoformat(self):
@@ -1215,7 +1215,7 @@
 
     __str__ = isoformat
 
-    def strftime(self, fmt):
+    def strftime(self, format):
         """Format using strftime().  The date part of the timestamp passed
         to underlying strftime should not be used.
         """
@@ -1224,13 +1224,13 @@
         timetuple = (1900, 1, 1,
                      self._hour, self._minute, self._second,
                      0, 1, -1)
-        return _wrap_strftime(self, fmt, timetuple)
+        return _wrap_strftime(self, format, timetuple)
 
-    def __format__(self, fmt):
-        if not isinstance(fmt, str):
-            raise TypeError("must be str, not %s" % type(fmt).__name__)
-        if len(fmt) != 0:
-            return self.strftime(fmt)
+    def __format__(self, format):
+        if not isinstance(format, str):
+            raise TypeError("must be str, not %s" % type(format).__name__)
+        if len(format) != 0:
+            return self.strftime(format)
         return str(self)
 
     # Timezone functions
diff --git a/lib-python/3/distutils/ccompiler.py b/lib-python/3/distutils/ccompiler.py
--- a/lib-python/3/distutils/ccompiler.py
+++ b/lib-python/3/distutils/ccompiler.py
@@ -959,7 +959,9 @@
 # is assumed to be in the 'distutils' package.)
 compiler_class = { 'unix':    ('unixccompiler', 'UnixCCompiler',
                                "standard UNIX-style compiler"),
-                   'msvc':    ('_msvccompiler', 'MSVCCompiler',
+                   # PyPy change: On win32 PyPy is translated with the
+                   # "msvc9compiler", force the same here.
+                   'msvc':    ('msvc9compiler', 'MSVCCompiler',
                                "Microsoft Visual C++"),
                    'cygwin':  ('cygwinccompiler', 'CygwinCCompiler',
                                "Cygwin port of GNU C Compiler for Win32"),
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -66,12 +66,12 @@
     so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
 
     g = {}
-    g['CC'] = "gcc -pthread"
-    g['CXX'] = "g++ -pthread"
+    g['CC'] = "cc -pthread"
+    g['CXX'] = "c++ -pthread"
     g['OPT'] = "-DNDEBUG -O2"
     g['CFLAGS'] = "-DNDEBUG -O2"
     g['CCSHARED'] = "-fPIC"
-    g['LDSHARED'] = "gcc -pthread -shared"
+    g['LDSHARED'] = "cc -pthread -shared"
     g['EXT_SUFFIX'] = so_ext
     g['SHLIB_SUFFIX'] = so_ext
     g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
diff --git a/lib-python/3/pathlib.py b/lib-python/3/pathlib.py
--- a/lib-python/3/pathlib.py
+++ b/lib-python/3/pathlib.py
@@ -1209,23 +1209,19 @@
     def mkdir(self, mode=0o777, parents=False, exist_ok=False):
         if self._closed:
             self._raise_closed()
-        if not parents:
-            try:
-                self._accessor.mkdir(self, mode)
-            except FileExistsError:
-                if not exist_ok or not self.is_dir():
-                    raise
-        else:
-            try:
-                self._accessor.mkdir(self, mode)
-            except FileExistsError:
-                if not exist_ok or not self.is_dir():
-                    raise
-            except OSError as e:
-                if e.errno != ENOENT:
-                    raise
-                self.parent.mkdir(parents=True)
-                self._accessor.mkdir(self, mode)
+        # NOTE: version from CPython 3.7 plus fix at issue #29694
+        try:
+            self._accessor.mkdir(self, mode)
+        except FileNotFoundError:
+            if not parents or self.parent == self:
+                raise
+            self.parent.mkdir(parents=True, exist_ok=True)
+            self.mkdir(mode, parents=False, exist_ok=exist_ok)
+        except OSError:
+            # Cannot rely on checking for EEXIST, since the operating system
+            # could give priority to other errors like EACCES or EROFS
+            if not exist_ok or not self.is_dir():
+                raise
 
     def chmod(self, mode):
         """
diff --git a/lib-python/3/site.py b/lib-python/3/site.py
--- a/lib-python/3/site.py
+++ b/lib-python/3/site.py
@@ -305,9 +305,7 @@
         seen.add(prefix)
 
         if is_pypy:
-            from distutils.sysconfig import get_python_lib
-            sitepackages.append(get_python_lib(standard_lib=False,
-                                               prefix=prefix))
+            sitepackages.append(os.path.join(prefix, "site-packages"))
         elif os.sep == '/':
             sitepackages.append(os.path.join(prefix, "lib",
                                         "python" + sys.version[:3],
@@ -410,7 +408,9 @@
             readline.parse_and_bind('tab: complete')
 
         try:
-            readline.read_init_file()
+            # Unimplemented on PyPy
+            #readline.read_init_file()
+            pass
         except OSError:
             # An OSError here could have many causes, but the most likely one
             # is that there's no .inputrc file (or .editrc file in the case of
diff --git a/lib-python/3/sre_compile.py b/lib-python/3/sre_compile.py
--- a/lib-python/3/sre_compile.py
+++ b/lib-python/3/sre_compile.py
@@ -378,7 +378,11 @@
 
 def _bytes_to_codes(b):
     # Convert block indices to word array
-    a = memoryview(b).cast('I')
+    if _sre.CODESIZE == 2:
+        code = 'H'
+    else:
+        code = 'I'
+    a = memoryview(b).cast(code)
     assert a.itemsize == _sre.CODESIZE
     assert len(a) * a.itemsize == len(b)
     return a.tolist()
diff --git a/lib-python/3/test/_test_multiprocessing.py b/lib-python/3/test/_test_multiprocessing.py
--- a/lib-python/3/test/_test_multiprocessing.py
+++ b/lib-python/3/test/_test_multiprocessing.py
@@ -2147,7 +2147,8 @@
 
         # Because we are using xmlrpclib for serialization instead of
         # pickle this will cause a serialization error.
-        self.assertRaises(Exception, queue.put, time.sleep)
+        # Changed on PyPy: passing functions to xmlrpc is broken
+        #self.assertRaises(Exception, queue.put, time.sleep)
 
         # Make queue finalizer run before the server is stopped
         del queue
diff --git a/lib-python/3/test/datetimetester.py b/lib-python/3/test/datetimetester.py
--- a/lib-python/3/test/datetimetester.py
+++ b/lib-python/3/test/datetimetester.py
@@ -1238,6 +1238,9 @@
         #check that this standard extension works
         t.strftime("%f")
 
+        #test that passing keyword arguments work
+        self.assertEqual(t.strftime(format=""), "")
+
     def test_format(self):
         dt = self.theclass(2007, 9, 10)
         self.assertEqual(dt.__format__(''), str(dt))
@@ -1602,6 +1605,9 @@
             self.assertEqual(a.__format__(fmt), dt.strftime(fmt))
             self.assertEqual(b.__format__(fmt), 'B')
 
+        #test that passing keyword arguments work
+        self.assertEqual(dt.strftime(format=""), "")
+
     def test_more_ctime(self):
         # Test fields that TestDate doesn't touch.
         import time
@@ -2335,6 +2341,9 @@
         # A naive object replaces %z and %Z with empty strings.
         self.assertEqual(t.strftime("'%z' '%Z'"), "'' ''")
 
+        #test that passing keyword arguments work
+        self.assertEqual(t.strftime(format=""), "")
+
     def test_format(self):
         t = self.theclass(1, 2, 3, 4)
         self.assertEqual(t.__format__(''), str(t))
diff --git a/lib-python/3/test/test_pathlib.py b/lib-python/3/test/test_pathlib.py
--- a/lib-python/3/test/test_pathlib.py
+++ b/lib-python/3/test/test_pathlib.py
@@ -8,6 +8,7 @@
 import stat
 import tempfile
 import unittest
+from unittest import mock
 
 from test import support
 TESTFN = support.TESTFN
@@ -1751,6 +1752,35 @@
             p.mkdir(exist_ok=True)
         self.assertEqual(cm.exception.errno, errno.EEXIST)
 
+    def test_mkdir_concurrent_parent_creation(self):
+        for pattern_num in range(32):
+            p = self.cls(BASE, 'dirCPC%d' % pattern_num)
+            self.assertFalse(p.exists())
+
+            def my_mkdir(path, mode=0o777):
+                path = str(path)
+                # Emulate another process that would create the directory
+                # just before we try to create it ourselves.  We do it
+                # in all possible pattern combinations, assuming that this
+                # function is called at most 5 times (dirCPC/dir1/dir2,
+                # dirCPC/dir1, dirCPC, dirCPC/dir1, cirCPC/dir1/dir2).
+                if pattern.pop():
+                    os.mkdir(path, mode)      # from another process
+                    concurrently_created.add(path)
+                os.mkdir(path, mode)          # our real call
+
+            pattern = [bool(pattern_num & (1 << n)) for n in range(5)]
+            concurrently_created = set()
+            p12 = p / 'dir1' / 'dir2'
+            try:
+                with mock.patch("pathlib._normal_accessor.mkdir", my_mkdir):
+                    p12.mkdir(parents=True, exist_ok=False)
+            except FileExistsError:
+                self.assertIn(str(p12), concurrently_created)
+            else:
+                self.assertNotIn(str(p12), concurrently_created)
+            self.assertTrue(p.exists())
+
     @with_symlinks
     def test_symlink_to(self):
         P = self.cls(BASE)
diff --git a/lib-python/3/test/test_socket.py b/lib-python/3/test/test_socket.py
--- a/lib-python/3/test/test_socket.py
+++ b/lib-python/3/test/test_socket.py
@@ -722,12 +722,12 @@
             s.sendto('\u2620', sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'str'", # cpython
-                       "'str' does not support the buffer interface"]) # pypy
+                       "a bytes-like object is required, not str"]) # pypy
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'complex'",
-                       "'complex' does not support the buffer interface"])
+                       "a bytes-like object is required, not complex"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', None)
         self.assertIn('NoneType', str(cm.exception))
@@ -736,12 +736,12 @@
             s.sendto('\u2620', 0, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'str'",
-                       "'str' does not support the buffer interface"])
+                       "a bytes-like object is required, not str"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(5j, 0, sockname)
         self.assertIn(str(cm.exception),
                       ["a bytes-like object is required, not 'complex'",
-                       "'complex' does not support the buffer interface"])
+                       "a bytes-like object is required, not complex"])
         with self.assertRaises(TypeError) as cm:
             s.sendto(b'foo', 0, None)
         self.assertIn('NoneType', str(cm.exception))
diff --git a/lib-python/3/test/test_xml_etree.py b/lib-python/3/test/test_xml_etree.py
--- a/lib-python/3/test/test_xml_etree.py
+++ b/lib-python/3/test/test_xml_etree.py
@@ -1878,6 +1878,17 @@
             with self.assertRaises(RuntimeError):
                 repr(e)  # Should not crash
 
+    def test_bad_find_returns_none(self):
+        # This behavior is the one we get historically when the C
+        # extension module is enabled.  With the Python version, it
+        # raised a TypeError instead.  There are projects out there
+        # that depend on the non-raising behavior, of course.
+        e = ET.Element('foo')
+        assert e.find('') is None
+        assert e.findall('') == []
+        assert e.findtext('') is None
+        assert e.findtext('', default="default.") == "default."
+
 class MutatingElementPath(str):
     def __new__(cls, elem, *args):
         self = str.__new__(cls, *args)
diff --git a/lib-python/3/xml/etree/ElementTree.py b/lib-python/3/xml/etree/ElementTree.py
--- a/lib-python/3/xml/etree/ElementTree.py
+++ b/lib-python/3/xml/etree/ElementTree.py
@@ -294,7 +294,14 @@
         Return the first matching element, or None if no element was found.
 
         """
-        return ElementPath.find(self, path, namespaces)
+        # Used to be:  return ElementPath.find(self, path, namespaces)
+        # but there are projects out there that rely on it getting None
+        # instead of an internal TypeError.  This is what the C version
+        # of this class does.
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return None
+        return next(result, None)
 
     def findtext(self, path, default=None, namespaces=None):
         """Find text for first matching element by tag name or path.
@@ -308,7 +315,17 @@
         content, the empty string is returned.
 
         """
-        return ElementPath.findtext(self, path, default, namespaces)
+        # Used to be:
+        #     return ElementPath.findtext(self, path, default, namespaces)
+        # See find().
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return default
+        try:
+            elem = next(result)
+            return elem.text or ""
+        except StopIteration:
+            return default
 
     def findall(self, path, namespaces=None):
         """Find all matching subelements by tag name or path.
@@ -319,7 +336,12 @@
         Returns list containing all matching elements in document order.
 
         """
-        return ElementPath.findall(self, path, namespaces)
+        # Used to be:  return ElementPath.findall(self, path, namespaces)
+        # See find().
+        result = ElementPath.iterfind(self, path, namespaces)
+        if result is None:
+            return []
+        return list(result)
 
     def iterfind(self, path, namespaces=None):
         """Find all matching subelements by tag name or path.
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -597,7 +597,7 @@
                     py.test.skip("%s module not included in %s" % (mod,
                                                                    execpath))
 
-            cmd = "%s %s %s" % (execpath, regrrun, fspath.purebasename)
+            cmd = "%s -m test -v %s" % (execpath, fspath.purebasename)
             # add watchdog for timing out
             cmd = "%s %s %s %s" % (python, watchdog_script, TIMEOUT, cmd)
         else:
diff --git a/lib_pypy/_audioop_build.py b/lib_pypy/_audioop_build.py
--- a/lib_pypy/_audioop_build.py
+++ b/lib_pypy/_audioop_build.py
@@ -656,4 +656,4 @@
 ffi.set_source("_audioop_cffi", C_SOURCE)
 
 if __name__ == "__main__":
-    ffi.compile()
+    ffi.compile(verbose=2)
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -5,16 +5,22 @@
 it renames the compiled shared object to _pypy_openssl.so (which means
 that cryptography can ship their own cffi backend)
 
+NOTE: currently, we have changed ``_cffi_src/openssl/callbacks.py`` to
+not rely on the CPython C API.
+
 # Tests?
 
 Currently this module is tested using CPython's standard library test suite.
 
 # Install it into PyPy's source tree
 
-Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command:
+Copy over all the sources into the folder `lib_pypy/_cffi_ssl/*`. Updating the cffi backend can be simply done by the following command::
 
     $ cp -r <cloned cryptography folder>/src/_cffi_src/* .
 
+NOTE: you need to keep our version of ``_cffi_src/openssl/callbacks.py``
+for now!
+
 # Crpytography version
 
 Copied over release version `1.7.2`
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/callbacks.py
@@ -13,8 +13,6 @@
 #include <openssl/x509.h>
 #include <openssl/x509_vfy.h>
 #include <openssl/crypto.h>
-
-#include <pythread.h>
 """
 
 TYPES = """
@@ -64,8 +62,50 @@
    Copyright 2001-2016 Python Software Foundation; All Rights Reserved.
 */
 
+#ifdef _WIN32
+#ifdef _MSC_VER
+#ifdef inline
+#undef inline
+#endif
+#define inline __inline
+#endif
+#include <Windows.h>
+typedef CRITICAL_SECTION mutex1_t;
+static inline void mutex1_init(mutex1_t *mutex) {
+    InitializeCriticalSection(mutex);
+}
+static inline void mutex1_lock(mutex1_t *mutex) {
+    EnterCriticalSection(mutex);
+}
+static inline void mutex1_unlock(mutex1_t *mutex) {
+    LeaveCriticalSection(mutex);
+}
+#else
+#include <stdio.h>
+#include <stdlib.h>
+#include <pthread.h>
+typedef pthread_mutex_t mutex1_t;
+#define ASSERT_STATUS(call)                             \
+    if (call != 0) {                                    \
+        perror("Fatal error in _cffi_ssl: " #call);     \
+        abort();                                        \
+    }
+static inline void mutex1_init(mutex1_t *mutex) {
+#if !defined(pthread_mutexattr_default)
+#  define pthread_mutexattr_default ((pthread_mutexattr_t *)NULL)
+#endif
+    ASSERT_STATUS(pthread_mutex_init(mutex, pthread_mutexattr_default));
+}
+static inline void mutex1_lock(mutex1_t *mutex) {
+    ASSERT_STATUS(pthread_mutex_lock(mutex));
+}
+static inline void mutex1_unlock(mutex1_t *mutex) {
+    ASSERT_STATUS(pthread_mutex_unlock(mutex));
+}
+#endif
+
 static unsigned int _ssl_locks_count = 0;
-static PyThread_type_lock *_ssl_locks = NULL;
+static mutex1_t *_ssl_locks = NULL;
 
 static void _ssl_thread_locking_function(int mode, int n, const char *file,
                                          int line) {
@@ -89,35 +129,32 @@
     }
 
     if (mode & CRYPTO_LOCK) {
-        PyThread_acquire_lock(_ssl_locks[n], 1);
+        mutex1_lock(_ssl_locks + n);
     } else {
-        PyThread_release_lock(_ssl_locks[n]);
+        mutex1_unlock(_ssl_locks + n);
+    }
+}
+
+static void init_mutexes(void)
+{
+    int i;
+    for (i = 0;  i < _ssl_locks_count;  i++) {
+        mutex1_init(_ssl_locks + i);
     }
 }
 
 int _setup_ssl_threads(void) {
-    unsigned int i;
-
     if (_ssl_locks == NULL) {
         _ssl_locks_count = CRYPTO_num_locks();
-        _ssl_locks = PyMem_New(PyThread_type_lock, _ssl_locks_count);
+        _ssl_locks = malloc(sizeof(mutex1_t) * _ssl_locks_count);
         if (_ssl_locks == NULL) {
-            PyErr_NoMemory();
             return 0;
         }
-        memset(_ssl_locks, 0, sizeof(PyThread_type_lock) * _ssl_locks_count);
-        for (i = 0;  i < _ssl_locks_count;  i++) {
-            _ssl_locks[i] = PyThread_allocate_lock();
-            if (_ssl_locks[i] == NULL) {
-                unsigned int j;
-                for (j = 0;  j < i;  j++) {
-                    PyThread_free_lock(_ssl_locks[j]);
-                }
-                PyMem_Free(_ssl_locks);
-                return 0;
-            }
-        }
+        init_mutexes();
         CRYPTO_set_locking_callback(_ssl_thread_locking_function);
+#ifndef _WIN32
+        pthread_atfork(NULL, NULL, &init_mutexes);
+#endif
     }
     return 1;
 }
diff --git a/lib_pypy/_cffi_ssl/_stdssl/__init__.py b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
--- a/lib_pypy/_cffi_ssl/_stdssl/__init__.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/__init__.py
@@ -46,7 +46,7 @@
 SERVER = 1
 
 VERIFY_DEFAULT = 0
-VERIFY_CRL_CHECK_LEAF = lib.X509_V_FLAG_CRL_CHECK 
+VERIFY_CRL_CHECK_LEAF = lib.X509_V_FLAG_CRL_CHECK
 VERIFY_CRL_CHECK_CHAIN = lib.X509_V_FLAG_CRL_CHECK | lib.X509_V_FLAG_CRL_CHECK_ALL
 VERIFY_X509_STRICT = lib.X509_V_FLAG_X509_STRICT
 if lib.Cryptography_HAS_X509_V_FLAG_TRUSTED_FIRST:
@@ -297,6 +297,10 @@
         else:
             raise TypeError("The value must be a SSLContext")
 
+    @property
+    def server_side(self):
+        return self.socket_type == SSL_SERVER
+
     def do_handshake(self):
         sock = self.get_socket_or_connection_gone()
         ssl = self.ssl
@@ -658,7 +662,7 @@
     def tls_unique_cb(self):
         buf = ffi.new("char[]", SSL_CB_MAXLEN)
 
-        if lib.SSL_session_reused(self.ssl) ^ (not self.socket_type):
+        if lib.SSL_session_reused(self.ssl) ^ (not self.server_side):
             # if session is resumed XOR we are the client
             length = lib.SSL_get_finished(self.ssl, buf, SSL_CB_MAXLEN)
         else:
@@ -1297,7 +1301,7 @@
     result = _asn1obj2py(obj);
     lib.ASN1_OBJECT_free(obj);
     return result;
-                                                               
+
 
 class MemoryBIO(object):
     def __init__(self):
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -83,8 +83,9 @@
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
-        res._base = base
-        res._index = index
+        if base is not None:
+            res._base = base
+            res._index = index
         return res
 
     def _CData_retval(self, resbuffer):
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -64,8 +64,9 @@
         res = object.__new__(self)
         res.__class__ = self
         res.__dict__['_buffer'] = resbuffer
-        res.__dict__['_base'] = base
-        res.__dict__['_index'] = index
+        if base is not None:
+            res.__dict__['_base'] = base
+            res.__dict__['_index'] = index
         return res
 
     def _CData_retval(self, resbuffer):
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -1,4 +1,3 @@
-
 from _ctypes.basics import _CData, _CDataMeta, cdata_from_address
 from _ctypes.primitive import SimpleType, _SimpleCData
 from _ctypes.basics import ArgumentError, keepalive_key
@@ -9,13 +8,16 @@
 import sys
 import traceback
 
-try: from __pypy__ import builtinify
-except ImportError: builtinify = lambda f: f
+
+try:
+    from __pypy__ import builtinify
+except ImportError:
+    builtinify = lambda f: f
 
 # XXX this file needs huge refactoring I fear
 
-PARAMFLAG_FIN   = 0x1
-PARAMFLAG_FOUT  = 0x2
+PARAMFLAG_FIN = 0x1
+PARAMFLAG_FOUT = 0x2
 PARAMFLAG_FLCID = 0x4
 PARAMFLAG_COMBINED = PARAMFLAG_FIN | PARAMFLAG_FOUT | PARAMFLAG_FLCID
 
@@ -24,7 +26,7 @@
     PARAMFLAG_FIN,
     PARAMFLAG_FIN | PARAMFLAG_FOUT,
     PARAMFLAG_FIN | PARAMFLAG_FLCID
-    )
+)
 
 WIN64 = sys.platform == 'win32' and sys.maxsize == 2**63 - 1
 
@@ -35,6 +37,7 @@
     from _ctypes import COMError
     return COMError(errcode, None, None)
 
+
 @builtinify
 def call_function(func, args):
     "Only for debugging so far: So that we can call CFunction instances"
@@ -93,14 +96,9 @@
                         "item %d in _argtypes_ has no from_param method" % (
                             i + 1,))
             self._argtypes_ = list(argtypes)
-            self._check_argtypes_for_fastpath()
+
     argtypes = property(_getargtypes, _setargtypes)
 
-    def _check_argtypes_for_fastpath(self):
-        if all([hasattr(argtype, '_ffiargshape_') for argtype in self._argtypes_]):
-            fastpath_cls = make_fastpath_subclass(self.__class__)
-            fastpath_cls.enable_fastpath_maybe(self)
-
     def _getparamflags(self):
         return self._paramflags
 
@@ -125,27 +123,26 @@
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             if not isinstance(flag, int):
                 raise TypeError(
                     "paramflags must be a sequence of (int [,string [,value]]) "
                     "tuples"
-                    )
+                )
             _flag = flag & PARAMFLAG_COMBINED
             if _flag == PARAMFLAG_FOUT:
                 typ = self._argtypes_[idx]
                 if getattr(typ, '_ffiargshape_', None) not in ('P', 'z', 'Z'):
                     raise TypeError(
                         "'out' parameter %d must be a pointer type, not %s"
-                        % (idx+1, type(typ).__name__)
-                        )
+                        % (idx + 1, type(typ).__name__)
+                    )
             elif _flag not in VALID_PARAMFLAGS:
                 raise TypeError("paramflag value %d not supported" % flag)
         self._paramflags = paramflags
 
     paramflags = property(_getparamflags, _setparamflags)
 
-
     def _getrestype(self):
         return self._restype_
 
@@ -155,7 +152,7 @@
             from ctypes import c_int
             restype = c_int
         if not (isinstance(restype, _CDataMeta) or restype is None or
-                callable(restype)):
+                    callable(restype)):
             raise TypeError("restype must be a type, a callable, or None")
         self._restype_ = restype
 
@@ -167,15 +164,18 @@
 
     def _geterrcheck(self):
         return getattr(self, '_errcheck_', None)
+
     def _seterrcheck(self, errcheck):
         if not callable(errcheck):
             raise TypeError("The errcheck attribute must be callable")
         self._errcheck_ = errcheck
+
     def _delerrcheck(self):
         try:
             del self._errcheck_
         except AttributeError:
             pass
+
     errcheck = property(_geterrcheck, _seterrcheck, _delerrcheck)
 
     def _ffishapes(self, args, restype):
@@ -187,7 +187,7 @@
                 raise TypeError("invalid result type for callback function")
             restype = restype._ffiargshape_
         else:
-            restype = 'O' # void
+            restype = 'O'  # void
         return argtypes, restype
 
     def _set_address(self, address):
@@ -200,7 +200,7 @@
 
     def __init__(self, *args):
         self.name = None
-        self._objects = {keepalive_key(0):self}
+        self._objects = {keepalive_key(0): self}
         self._needs_free = True
 
         # Empty function object -- this is needed for casts
@@ -221,10 +221,8 @@
             if self._argtypes_ is None:
                 self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
-            self._check_argtypes_for_fastpath()
             return
 
-
         # A callback into python
         if callable(argument) and not argsl:
             self.callable = argument
@@ -258,7 +256,7 @@
         if (sys.platform == 'win32' and isinstance(argument, int)
             and argsl):
             ffiargs, ffires = self._ffishapes(self._argtypes_, self._restype_)
-            self._com_index =  argument + 0x1000
+            self._com_index = argument + 0x1000
             self.name = argsl.pop(0)
             if argsl:
                 self.paramflags = argsl.pop(0)
@@ -280,6 +278,7 @@
             except SystemExit as e:
                 handle_system_exit(e)
                 raise
+
         return f
 
     def __call__(self, *args, **kwargs):
@@ -328,7 +327,7 @@
             # really slow".  Now we don't worry that much about slowness
             # of ctypes, and it's strange to get warnings for perfectly-
             # legal code.
-            #warnings.warn('C function without declared arguments called',
+            # warnings.warn('C function without declared arguments called',
             #              RuntimeWarning, stacklevel=2)
             argtypes = []
 
@@ -337,7 +336,7 @@
             if not args:
                 raise ValueError(
                     "native COM method call without 'this' parameter"
-                    )
+                )
             thisvalue = args[0]
             thisarg = cast(thisvalue, POINTER(POINTER(c_void_p)))
             keepalives, newargs, argtypes, outargs, errcheckargs = (
@@ -366,7 +365,6 @@
         return tuple(outargs)
 
     def _call_funcptr(self, funcptr, *newargs):
-
         if self._flags_ & _rawffi.FUNCFLAG_USE_ERRNO:
             tmp = _rawffi.get_errno()
             _rawffi.set_errno(get_errno())
@@ -431,7 +429,7 @@
             ffiargs = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
-        
+
         cdll = self.dll._handle
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
@@ -450,7 +448,7 @@
             # funcname -> _funcname@<n>
             # where n is 0, 4, 8, 12, ..., 128
             for i in range(33):
-                mangled_name = "_%s@%d" % (self.name, i*4)
+                mangled_name = "_%s@%d" % (self.name, i * 4)
                 try:
                     return cdll.getfunc(mangled_name,
                                         ffi_argtypes, ffi_restype,
@@ -492,7 +490,7 @@
         for argtype, arg in zip(argtypes, args):
             param = argtype.from_param(arg)
             _type_ = getattr(argtype, '_type_', None)
-            if _type_ == 'P': # special-case for c_void_p
+            if _type_ == 'P':  # special-case for c_void_p
                 param = param._get_buffer_value()
             elif self._is_primitive(argtype):
                 param = param.value
@@ -668,69 +666,11 @@
             self._needs_free = False
 
 
-def make_fastpath_subclass(CFuncPtr):
-    if CFuncPtr._is_fastpath:
-        return CFuncPtr
-    #
-    try:
-        return make_fastpath_subclass.memo[CFuncPtr]
-    except KeyError:
-        pass
-
-    class CFuncPtrFast(CFuncPtr):
-
-        _is_fastpath = True
-        _slowpath_allowed = True # set to False by tests
-
-        @classmethod
-        def enable_fastpath_maybe(cls, obj):
-            if (obj.callable is None and
-                obj._com_index is None):
-                obj.__class__ = cls
-
-        def __rollback(self):
-            assert self._slowpath_allowed
-            self.__class__ = CFuncPtr
-
-        # disable the fast path if we reset argtypes
-        def _setargtypes(self, argtypes):
-            self.__rollback()
-            self._setargtypes(argtypes)
-        argtypes = property(CFuncPtr._getargtypes, _setargtypes)
-
-        def _setcallable(self, func):
-            self.__rollback()
-            self.callable = func
-        callable = property(lambda x: None, _setcallable)
-
-        def _setcom_index(self, idx):
-            self.__rollback()
-            self._com_index = idx
-        _com_index = property(lambda x: None, _setcom_index)
-
-        def __call__(self, *args):
-            thisarg = None
-            argtypes = self._argtypes_
-            restype = self._restype_
-            funcptr = self._getfuncptr(argtypes, restype, thisarg)
-            try:
-                result = self._call_funcptr(funcptr, *args)
-                result, _ = self._do_errcheck(result, args)
-            except (TypeError, ArgumentError, UnicodeDecodeError):
-                assert self._slowpath_allowed
-                return CFuncPtr.__call__(self, *args)
-            return result
-
-    make_fastpath_subclass.memo[CFuncPtr] = CFuncPtrFast
-    return CFuncPtrFast
-make_fastpath_subclass.memo = {}
-
-
 def handle_system_exit(e):
     # issue #1194: if we get SystemExit here, then exit the interpreter.
     # Highly obscure imho but some people seem to depend on it.
     if sys.flags.inspect:
-        return   # Don't exit if -i flag was given.
+        return  # Don't exit if -i flag was given.
     else:
         code = e.code
         if isinstance(code, int):
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -233,6 +233,9 @@
         if ('_abstract_' in cls.__dict__ or cls is Structure 
                                          or cls is union.Union):
             raise TypeError("abstract class")
+        if hasattr(cls, '_swappedbytes_'):
+            raise NotImplementedError("missing in PyPy: structure/union with "
+                                      "swapped (non-native) byte ordering")
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_decimal_build.py b/lib_pypy/_decimal_build.py
--- a/lib_pypy/_decimal_build.py
+++ b/lib_pypy/_decimal_build.py
@@ -226,6 +226,16 @@
 _libdir = os.path.join(os.path.dirname(__file__), '_libmpdec')
 ffi.set_source('_decimal_cffi',
     """
+#ifdef _MSC_VER
+  #if defined(_WIN64)
+    typedef __int64 LONG_PTR; 
+  #else
+    typedef long LONG_PTR;
+  #endif
+  typedef LONG_PTR ssize_t;
+#else
+  #define HAVE_STDINT_H
+#endif
 #include "mpdecimal.h"
 
 #define MPD_Float_operation MPD_Not_implemented
@@ -266,7 +276,6 @@
     include_dirs=[_libdir],
     extra_compile_args=[
         "-DANSI",
-        "-DHAVE_STDINT_H",
         "-DHAVE_INTTYPES_H",
         "-DCONFIG_64" if sys.maxsize > 1 << 32 else "-DCONFIG_32",
     ],
diff --git a/lib_pypy/_lzma_build.py b/lib_pypy/_lzma_build.py
--- a/lib_pypy/_lzma_build.py
+++ b/lib_pypy/_lzma_build.py
@@ -235,6 +235,9 @@
 """)
 
 ffi.set_source('_lzma_cffi', """
+#ifdef _MSC_VER
+#define LZMA_API_STATIC
+#endif
 #include <lzma.h>
 #include <stdlib.h>
 void _pylzma_stream_init(lzma_stream *strm) {
diff --git a/lib_pypy/_md5.py b/lib_pypy/_md5.py
--- a/lib_pypy/_md5.py
+++ b/lib_pypy/_md5.py
@@ -53,10 +53,10 @@
     j = 0
     i = 0
     while i < imax:
-        b0 = ord(list[j])
-        b1 = ord(list[j+1]) << 8
-        b2 = ord(list[j+2]) << 16
-        b3 = ord(list[j+3]) << 24
+        b0 = list[j]
+        b1 = list[j+1] << 8
+        b2 = list[j+2] << 16
+        b3 = list[j+3] << 24
         hl[i] = b0 | b1 |b2 | b3
         i = i+1
         j = j+4
@@ -319,7 +319,7 @@
         else:
             padLen = 120 - index
 
-        padding = [b'\200'] + [b'\000'] * 63
+        padding = [128] + [0] * 63
         self.update(padding[:padLen])
 
         # Append length (before padding).
diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -1,6 +1,5 @@
-from __pypy__ import reversed_dict, move_to_end
+from __pypy__ import reversed_dict, move_to_end, objects_in_repr
 from _operator import eq as _eq
-from reprlib import recursive_repr as _recursive_repr
 import _collections_abc
 
 
@@ -44,12 +43,21 @@
         '''
         return move_to_end(self, key, last)
 
-    @_recursive_repr()
     def __repr__(self):
         'od.__repr__() <==> repr(od)'
         if not self:
             return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+        currently_in_repr = objects_in_repr()
+        if self in currently_in_repr:
+            return '...'
+        currently_in_repr[self] = 1
+        try:
+            return '%s(%r)' % (self.__class__.__name__, list(self.items()))
+        finally:
+            try:
+                del currently_in_repr[self]
+            except:
+                pass
 
     def __reduce__(self):
         'Return state information for pickling'
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -61,13 +61,12 @@
     # set link options
     output_filename = modulename + _get_c_extension_suffix()
     if sys.platform == 'win32':
-        # XXX pyconfig.h uses a pragma to link to the import library,
-        #     which is currently python3.lib
-        library = os.path.join(thisdir, '..', 'libs', 'python32')
+        libname = 'python{0[0]}{0[1]}'.format(sys.version_info)
+        library = os.path.join(thisdir, '..', 'lib', libname)
         if not os.path.exists(library + '.lib'):
             # For a local translation or nightly build
-            library = os.path.join(thisdir, '..', 'pypy', 'goal', 'python32')
-        assert os.path.exists(library + '.lib'),'Could not find import library "%s"' % library
+            library = os.path.join(thisdir, '..', 'pypy', 'goal', libname)
+        assert os.path.exists(library + '.lib'), 'Could not find import library "%s"' % library
         libraries = [library, 'oleaut32']
         extra_ldargs = ['/MANIFEST',  # needed for VC10
                         '/EXPORT:PyInit_' + modulename]
diff --git a/lib_pypy/_pypy_winbase_build.py b/lib_pypy/_pypy_winbase_build.py
--- a/lib_pypy/_pypy_winbase_build.py
+++ b/lib_pypy/_pypy_winbase_build.py
@@ -76,6 +76,9 @@
 HANDLE WINAPI GetCurrentProcess(void);
 BOOL WINAPI DuplicateHandle(HANDLE, HANDLE, HANDLE, LPHANDLE,
                             DWORD, BOOL, DWORD);
+BOOL WINAPI CreateProcessA(char *, char *, void *,
+                           void *, BOOL, DWORD, char *,
+                           char *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
 BOOL WINAPI CreateProcessW(wchar_t *, wchar_t *, void *,
                            void *, BOOL, DWORD, wchar_t *,
                            wchar_t *, LPSTARTUPINFO, LPPROCESS_INFORMATION);
@@ -84,6 +87,12 @@
 BOOL WINAPI TerminateProcess(HANDLE, UINT);
 HANDLE WINAPI GetStdHandle(DWORD);
 DWORD WINAPI GetModuleFileNameW(HANDLE, wchar_t *, DWORD);
+
+UINT WINAPI SetErrorMode(UINT);
+#define SEM_FAILCRITICALERRORS     0x0001
+#define SEM_NOGPFAULTERRORBOX      0x0002
+#define SEM_NOALIGNMENTFAULTEXCEPT 0x0004
+#define SEM_NOOPENFILEERRORBOX     0x8000
 """)
 
 # --------------------
diff --git a/lib_pypy/_pypy_winbase_cffi.py b/lib_pypy/_pypy_winbase_cffi.py
--- a/lib_pypy/_pypy_winbase_cffi.py
+++ b/lib_pypy/_pypy_winbase_cffi.py
@@ -3,8 +3,8 @@
 
 ffi = _cffi_backend.FFI('_pypy_winbase_cffi',
     _version = 0x2601,
-    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x13\x11\x00\x00\x59\x03\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x16\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x13\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x4C\x03\x00\x00\x2D\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x2D\x11\x00\x00\x2D\x11\x00\x00\x54\x03\x00\x00\x53\x03\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x15\x11\x00\x00\x2D\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x16\x0D\x00\x00\x02\x0F\x00\x00\x47\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x00\x0F\x00\x00\x47\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x56\x03\x00\x00\x02\x01\x00\x00\x58\x03\x00\x00\x04\x01\x00\x00\x00\x01',
-    _globals = (b'\x00\x00\x18\x23CloseHandle',0,b'\x00\x00\x12\x23CreatePipe',0,b'\x00\x00\x2C\x23CreateProcessW',0,b'\x00\x00\x23\x23DuplicateHandle',0,b'\x00\x00\x51\x23GetCurrentProcess',0,b'\x00\x00\x1F\x23GetExitCodeProcess',0,b'\x00\x00\x3F\x23GetModuleFileNameW',0,b'\x00\x00\x4E\x23GetStdHandle',0,b'\x00\x00\x44\x23GetVersion',0,b'\x00\x00\x1B\x23TerminateProcess',0,b'\x00\x00\x3B\x23WaitForSingleObject',0,b'\x00\x00\x38\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x49\x23_getwch',0,b'\x00\x00\x49\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x4B\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x46\x23_ungetwch',0),
-    _struct_unions = ((b'\x00\x00\x00\x53\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x16\x11dwProcessId',b'\x00\x00\x16\x11dwThreadId'),(b'\x00\x00\x00\x54\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x16\x11cb',b'\x00\x00\x55\x11lpReserved',b'\x00\x00\x55\x11lpDesktop',b'\x00\x00\x55\x11lpTitle',b'\x00\x00\x16\x11dwX',b'\x00\x00\x16\x11dwY',b'\x00\x00\x16\x11dwXSize',b'\x00\x00\x16\x11dwYSize',b'\x00\x00\x16\x11dwXCountChars',b'\x00\x00\x16\x11dwYCountChars',b'\x00\x00\x16\x11dwFillAttribute',b'\x00\x00\x16\x11dwFlags',b'\x00\x00\x47\x11wShowWindow',b'\x00\x00\x47\x11cbReserved2',b'\x00\x00\x57\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
-    _typenames = (b'\x00\x00\x00\x36LPPROCESS_INFORMATION',b'\x00\x00\x00\x35LPSTARTUPINFO',b'\x00\x00\x00\x53PROCESS_INFORMATION',b'\x00\x00\x00\x54STARTUPINFO',b'\x00\x00\x00\x47wint_t'),
+    _types = b'\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x07\x01\x00\x00\x07\x01\x00\x00\x09\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x19\x01\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x00\x0F\x00\x00\x01\x0D\x00\x00\x64\x03\x00\x00\x13\x11\x00\x00\x67\x03\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x13\x11\x00\x00\x13\x11\x00\x00\x63\x03\x00\x00\x62\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x03\x00\x00\x1F\x11\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x18\x03\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x1F\x11\x00\x00\x0A\x01\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x01\x0D\x00\x00\x5B\x03\x00\x00\x39\x11\x00\x00\x15\x11\x00\x00\x15\x11\x00\x00\x07\x01\x00\x00\x0A\x01\x00\x00\x39\x11\x00\x00\x39\x11\x00\x00\x1B\x11\x00\x00\x1C\x11\x00\x00\x02\x0F\x00\x00\x0D\x0D\x00\x00\x07\x01\x00\x00\x00\x0F\x00\x00\x29\x0D\x00\x00\x08\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x15\x11\x00\x00\x39\x11\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x18\x0D\x00\x00\x02\x0F\x00\x00\x56\x0D\x00\x00\x06\x01\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x00\x0F\x00\x00\x56\x0D\x00\x00\x10\x01\x00\x00\x00\x0F\x00\x00\x15\x0D\x00\x00\x0A\x01\x00\x00\x02\x0F\x00\x00\x15\x0D\x00\x00\x02\x0F\x00\x00\x00\x09\x00\x00\x01\x09\x00\x00\x02\x01\x00\x00\x66\x03\x00\x00\x04\x01\x00\x00\x00\x01',
+    _globals = (b'\x00\x00\x24\x23CloseHandle',0,b'\x00\x00\x1E\x23CreatePipe',0,b'\x00\x00\x12\x23CreateProcessA',0,b'\x00\x00\x38\x23CreateProcessW',0,b'\x00\x00\x2F\x23DuplicateHandle',0,b'\x00\x00\x60\x23GetCurrentProcess',0,b'\x00\x00\x2B\x23GetExitCodeProcess',0,b'\x00\x00\x4E\x23GetModuleFileNameW',0,b'\x00\x00\x5D\x23GetStdHandle',0,b'\x00\x00\x53\x23GetVersion',0,b'\xFF\xFF\xFF\x1FSEM_FAILCRITICALERRORS',1,b'\xFF\xFF\xFF\x1FSEM_NOALIGNMENTFAULTEXCEPT',4,b'\xFF\xFF\xFF\x1FSEM_NOGPFAULTERRORBOX',2,b'\xFF\xFF\xFF\x1FSEM_NOOPENFILEERRORBOX',32768,b'\x00\x00\x47\x23SetErrorMode',0,b'\x00\x00\x27\x23TerminateProcess',0,b'\x00\x00\x4A\x23WaitForSingleObject',0,b'\x00\x00\x44\x23_get_osfhandle',0,b'\x00\x00\x10\x23_getch',0,b'\x00\x00\x10\x23_getche',0,b'\x00\x00\x58\x23_getwch',0,b'\x00\x00\x58\x23_getwche',0,b'\x00\x00\x10\x23_kbhit',0,b'\x00\x00\x07\x23_locking',0,b'\x00\x00\x0C\x23_open_osfhandle',0,b'\x00\x00\x00\x23_putch',0,b'\x00\x00\x5A\x23_putwch',0,b'\x00\x00\x03\x23_setmode',0,b'\x00\x00\x00\x23_ungetch',0,b'\x00\x00\x55\x23_ungetwch',0),
+    _struct_unions = ((b'\x00\x00\x00\x62\x00\x00\x00\x02$PROCESS_INFORMATION',b'\x00\x00\x15\x11hProcess',b'\x00\x00\x15\x11hThread',b'\x00\x00\x18\x11dwProcessId',b'\x00\x00\x18\x11dwThreadId'),(b'\x00\x00\x00\x63\x00\x00\x00\x02$STARTUPINFO',b'\x00\x00\x18\x11cb',b'\x00\x00\x13\x11lpReserved',b'\x00\x00\x13\x11lpDesktop',b'\x00\x00\x13\x11lpTitle',b'\x00\x00\x18\x11dwX',b'\x00\x00\x18\x11dwY',b'\x00\x00\x18\x11dwXSize',b'\x00\x00\x18\x11dwYSize',b'\x00\x00\x18\x11dwXCountChars',b'\x00\x00\x18\x11dwYCountChars',b'\x00\x00\x18\x11dwFillAttribute',b'\x00\x00\x18\x11dwFlags',b'\x00\x00\x56\x11wShowWindow',b'\x00\x00\x56\x11cbReserved2',b'\x00\x00\x65\x11lpReserved2',b'\x00\x00\x15\x11hStdInput',b'\x00\x00\x15\x11hStdOutput',b'\x00\x00\x15\x11hStdError')),
+    _typenames = (b'\x00\x00\x00\x1CLPPROCESS_INFORMATION',b'\x00\x00\x00\x1BLPSTARTUPINFO',b'\x00\x00\x00\x62PROCESS_INFORMATION',b'\x00\x00\x00\x63STARTUPINFO',b'\x00\x00\x00\x56wint_t'),
 )
diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -32,10 +32,11 @@
 import threading
 
 try:
-    from __pypy__ import newlist_hint
+    from __pypy__ import newlist_hint, add_memory_pressure
 except ImportError:
     assert '__pypy__' not in sys.builtin_module_names
     newlist_hint = lambda sizehint: []
+    add_memory_pressure = lambda size: None
 
 if sys.version_info[0] >= 3:
     StandardError = Exception
@@ -152,6 +153,9 @@
                  check_same_thread=True, factory=None, cached_statements=100,
                  uri=0):
     factory = Connection if not factory else factory
+    # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
+    # backed by :memory: or a file)
+    add_memory_pressure(100 * 1024)
     return factory(database, timeout, detect_types, isolation_level,
                     check_same_thread, factory, cached_statements, uri)
 
diff --git a/lib_pypy/_structseq.py b/lib_pypy/_structseq.py
--- a/lib_pypy/_structseq.py
+++ b/lib_pypy/_structseq.py
@@ -129,3 +129,38 @@
     parts = ["%s=%r" % (fields[index].__name__, value)
             for index, value in enumerate(self[:visible_count])]
     return "%s(%s)" % (self._name, ", ".join(parts))
+
+
+class SimpleNamespace:
+    """A simple attribute-based namespace.
+
+SimpleNamespace(**kwargs)"""
+
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+    def __repr__(self):
+        ident = id(self)
+        if ident in sns_recurse:
+            return "namespace(...)"
+        sns_recurse.add(ident)
+        try:
+            pairs = ('%s=%r' % item for item in sorted(self.__dict__.items()))
+            return "namespace(%s)" % ', '.join(pairs)
+        finally:
+            sns_recurse.discard(ident)
+
+    def __eq__(self, other):
+        if issubclass(type(other), SimpleNamespace):
+            return self.__dict__ == other.__dict__
+        return NotImplemented
+
+    def __ne__(self, other):
+        if issubclass(type(other), SimpleNamespace):
+            return self.__dict__ != other.__dict__
+        return NotImplemented
+
+sns_recurse = set()
+
+# This class is not exposed in sys, but by the types module.
+SimpleNamespace.__module__ = 'types'
diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -1,6 +1,6 @@
-import imp
+import _imp
 
-so_ext = [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+so_ext = _imp.extension_suffixes()[0]
 
 build_time_vars = {
     "EXT_SUFFIX": so_ext,
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -563,7 +563,7 @@
             if sys.platform == "win32":
                 # we need 'libpypy-c.lib'.  Current distributions of
                 # pypy (>= 4.1) contain it as 'libs/python27.lib'.
-                pythonlib = "python27"
+                pythonlib = "python{0[0]}{0[1]}".format(sys.version_info)
                 if hasattr(sys, 'prefix'):
                     ensure('library_dirs', os.path.join(sys.prefix, 'libs'))
             else:
diff --git a/lib_pypy/msvcrt.py b/lib_pypy/msvcrt.py
--- a/lib_pypy/msvcrt.py
+++ b/lib_pypy/msvcrt.py
@@ -16,6 +16,7 @@
 import _rawffi
 from _pypy_winbase_cffi import ffi as _ffi
 _lib = _ffi.dlopen(_rawffi.get_libc().name)
+_kernel32 = _ffi.dlopen('kernel32')
 
 import errno
 
@@ -122,3 +123,9 @@
 def ungetwch(ch):
     if _lib._ungetwch(ord(ch)) == -1:   # EOF
         _ioerr()
+
+SetErrorMode = _kernel32.SetErrorMode
+SEM_FAILCRITICALERRORS     = _lib.SEM_FAILCRITICALERRORS
+SEM_NOGPFAULTERRORBOX      = _lib.SEM_NOGPFAULTERRORBOX
+SEM_NOALIGNMENTFAULTEXCEPT = _lib.SEM_NOALIGNMENTFAULTEXCEPT
+SEM_NOOPENFILEERRORBOX     = _lib.SEM_NOOPENFILEERRORBOX
diff --git a/lib_pypy/pyrepl/commands.py b/lib_pypy/pyrepl/commands.py
--- a/lib_pypy/pyrepl/commands.py
+++ b/lib_pypy/pyrepl/commands.py
@@ -384,4 +384,7 @@
 class quoted_insert(Command):
     kills_digit_arg = 0
     def do(self):
-        self.reader.push_input_trans(QITrans())
+        # XXX in Python 3, processing insert/C-q/C-v keys crashes
+        # because of a mixture of str and bytes.  Disable these keys.
+        pass
+        #self.reader.push_input_trans(QITrans())
diff --git a/lib_pypy/pyrepl/keymaps.py b/lib_pypy/pyrepl/keymaps.py
--- a/lib_pypy/pyrepl/keymaps.py
+++ b/lib_pypy/pyrepl/keymaps.py
@@ -62,7 +62,7 @@
      (r'\M-\n', 'self-insert'),
      (r'\<backslash>', 'self-insert')] + \
     [(c, 'self-insert')
-     for c in map(chr, range(32, 127)) if c <> '\\'] + \
+     for c in map(chr, range(32, 127)) if c != '\\'] + \
     [(c, 'self-insert')
      for c in map(chr, range(128, 256)) if c.isalpha()] + \
     [(r'\<up>', 'up'),
@@ -101,7 +101,7 @@
     
 reader_vi_insert_keymap = tuple(
     [(c, 'self-insert')
-     for c in map(chr, range(32, 127)) if c <> '\\'] + \
+     for c in map(chr, range(32, 127)) if c != '\\'] + \
     [(c, 'self-insert')
      for c in map(chr, range(128, 256)) if c.isalpha()] + \
     [(r'\C-d', 'delete'),
diff --git a/lib_pypy/pyrepl/pygame_console.py b/lib_pypy/pyrepl/pygame_console.py
--- a/lib_pypy/pyrepl/pygame_console.py
+++ b/lib_pypy/pyrepl/pygame_console.py
@@ -130,7 +130,7 @@
         s.fill(c, [0, 600 - bmargin, 800, bmargin])
         s.fill(c, [800 - rmargin, 0, lmargin, 600])
 
-    def refresh(self, screen, (cx, cy)):
+    def refresh(self, screen, cxy):
         self.screen = screen
         self.pygame_screen.fill(colors.bg,
                                 [0, tmargin + self.cur_top + self.scroll,
@@ -139,6 +139,7 @@
 
         line_top = self.cur_top
         width, height = self.fontsize
+        cx, cy = cxy
         self.cxy = (cx, cy)
         cp = self.char_pos(cx, cy)
         if cp[1] < tmargin:
@@ -282,7 +283,7 @@
 
     def forgetinput(self):
         """Forget all pending, but not yet processed input."""
-        while pygame.event.poll().type <> NOEVENT:
+        while pygame.event.poll().type != NOEVENT:
             pass
     
     def getpending(self):
@@ -299,7 +300,7 @@
 
     def wait(self):
         """Wait for an event."""
-        raise Exception, "erp!"
+        raise Exception("erp!")
 
     def repaint(self):
         # perhaps we should consolidate grobs?
diff --git a/lib_pypy/pyrepl/pygame_keymap.py b/lib_pypy/pyrepl/pygame_keymap.py
--- a/lib_pypy/pyrepl/pygame_keymap.py
+++ b/lib_pypy/pyrepl/pygame_keymap.py
@@ -90,22 +90,22 @@
                 s += 2
             elif c == "c":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\C must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if ctrl:
-                    raise KeySpecError, "doubled \\C- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\C- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 ctrl = 1
                 s += 3
             elif c == "m":
                 if key[s + 2] != '-':
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "\\M must be followed by `-' (char %d of %s)"%(
-                        s + 2, repr(key))
+                        s + 2, repr(key)))
                 if meta:
-                    raise KeySpecError, "doubled \\M- (char %d of %s)"%(
-                        s + 1, repr(key))
+                    raise KeySpecError("doubled \\M- (char %d of %s)"%(
+                        s + 1, repr(key)))
                 meta = 1
                 s += 3
             elif c.isdigit():
@@ -119,22 +119,22 @@
             elif c == '<':
                 t = key.find('>', s)
                 if t == -1:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unterminated \\< starting at char %d of %s"%(
-                        s + 1, repr(key))
+                        s + 1, repr(key)))
                 try:
                     ret = _keynames[key[s+2:t].lower()]
                     s = t + 1
                 except KeyError:
-                    raise KeySpecError, \
+                    raise KeySpecError(
                               "unrecognised keyname `%s' at char %d of %s"%(
-                        key[s+2:t], s + 2, repr(key))
+                        key[s+2:t], s + 2, repr(key)))
                 if ret is None:
                     return None, s
             else:
-                raise KeySpecError, \
+                raise KeySpecError(
                           "unknown backslash escape %s at char %d of %s"%(
-                    `c`, s + 2, repr(key))
+                    repr(c), s + 2, repr(key)))
         else:
             if ctrl:
                 ret = chr(ord(key[s]) & 0x1f)   # curses.ascii.ctrl()
@@ -160,9 +160,9 @@
         r.setdefault(key[0], {})[key[1:]] = value
     for key, value in r.items():
         if value.has_key(()):
-            if len(value) <> 1:
-                raise KeySpecError, \
-                          "key definitions for %s clash"%(value.values(),)
+            if len(value) != 1:
+                raise KeySpecError(
+                          "key definitions for %s clash"%(value.values(),))
             else:
                 r[key] = value[()]
         else:
@@ -202,7 +202,7 @@
         return ''
     name, s = keyname(keyseq)
     if name:
-        if name <> 'escape' or s == len(keyseq):
+        if name != 'escape' or s == len(keyseq):
             return '\\<' + name + '>' + unparse_key(keyseq[s:])
         else:
             return '\\M-' + unparse_key(keyseq[1:])
@@ -226,7 +226,7 @@
         return []
     name, s = keyname(keyseq)
     if name:
-        if name <> 'escape' or s == len(keyseq):
+        if name != 'escape' or s == len(keyseq):
             return [name] + _unparse_keyf(keyseq[s:])
         else:
             rest = _unparse_keyf(keyseq[1:])
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -46,8 +46,9 @@
 from rpython.jit.backend import detect_cpu
 try:
     if detect_cpu.autodetect().startswith('x86'):
-        working_modules.add('_vmprof')
-        working_modules.add('faulthandler')
+        if not sys.platform.startswith('openbsd'):
+            working_modules.add('_vmprof')
+            working_modules.add('faulthandler')
 except detect_cpu.ProcessorAutodetectError:
     pass
 
@@ -72,8 +73,6 @@
 
     if "cppyy" in working_modules:
         working_modules.remove("cppyy")  # not tested on win32
-    if "faulthandler" in working_modules:
-        working_modules.remove("faulthandler")  # missing details
 
     # The _locale module is needed by site.py on Windows
     default_modules.add("_locale")
@@ -224,9 +223,6 @@
         BoolOption("withsmalllong", "use a version of 'long' in a C long long",
                    default=False),
 
-        BoolOption("withstrbuf", "use strings optimized for addition (ver 2)",
-                   default=False),
-
         BoolOption("withspecialisedtuple",
                    "use specialised tuples",
                    default=False),
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -79,6 +79,9 @@
 _ssl
     libssl
 
+_vmprof
+    libunwind (optional, loaded dynamically at runtime)
+
 Make sure to have these libraries (with development headers) installed
 before building PyPy, otherwise the resulting binary will not contain
 these modules.  Furthermore, the following libraries should be present
@@ -107,22 +110,22 @@
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
-    tk-dev libgc-dev \
+    tk-dev libgc-dev python-cffi \
     liblzma-dev  # For lzma on PyPy3.
 
 On Fedora::
 
     dnf install gcc make libffi-devel pkgconfig zlib-devel bzip2-devel \
     sqlite-devel ncurses-devel expat-devel openssl-devel tk-devel \
-    gdbm-devel \
+    gdbm-devel python-cffi\
     xz-devel  # For lzma on PyPy3.
 
 On SLES11::
 
     zypper install gcc make python-devel pkg-config \
     zlib-devel libopenssl-devel libbz2-devel sqlite3-devel \
-    libexpat-devel libffi-devel python-curses \
-    xz-devel  # For lzma on PyPy3.
+    libexpat-devel libffi-devel python-curses python-cffi \
+    xz-devel # For lzma on PyPy3.
     (XXX plus the SLES11 version of libgdbm-dev and tk-dev)
 
 On Mac OS X, most of these build-time dependencies are installed alongside
@@ -185,7 +188,7 @@
 ::
 
     cd pypy/tool/release
-    ./package.py pypy-VER-PLATFORM
+    ./package.py --archive-name=pypy-VER-PLATFORM
 
 This creates a clean and prepared hierarchy, as well as a ``.tar.bz2``
 with the same content; both are found by default in
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -364,6 +364,24 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+Performance Differences
+-------------------------
+
+CPython has an optimization that can make repeated string concatenation not
+quadratic. For example, this kind of code runs in O(n) time::
+
+    s = ''
+    for string in mylist:
+        s += string
+
+In PyPy, this code will always have quadratic complexity. Note also, that the
+CPython optimization is brittle and can break by having slight variations in
+your code anyway. So you should anyway replace the code with::
+
+    parts = []
+    for string in mylist:
+        parts.append(string)
+    s = "".join(parts)
 
 Miscellaneous
 -------------
@@ -496,7 +514,14 @@
   the rest is kept.  If you return an unexpected string from
   ``__hex__()`` you get an exception (or a crash before CPython 2.7.13).
 
-* PyPy3: ``__class__`` attritube assignment between heaptypes and non heaptypes.
+* In PyPy, dictionaries passed as ``**kwargs`` can contain only string keys,
+  even for ``dict()`` and ``dict.update()``.  CPython 2.7 allows non-string
+  keys in these two cases (and only there, as far as we know).  E.g. this
+  code produces a ``TypeError``, on CPython 3.x as well as on any PyPy:
+  ``dict(**{1: 2})``.  (Note that ``dict(**d1)`` is equivalent to
+  ``dict(d1)``.)
+
+* PyPy3: ``__class__`` attribute assignment between heaptypes and non heaptypes.
   CPython allows that for module subtypes, but not for e.g. ``int``
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.7.1.rst
    release-v5.7.0.rst
    release-pypy2.7-v5.6.0.rst
    release-pypy2.7-v5.4.1.rst
@@ -59,6 +60,7 @@
 
 .. toctree::
 
+   release-v5.7.1.rst
    release-v5.7.0.rst
 
 CPython 3.3 compatible versions
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -32,10 +32,10 @@
 
 .. code-block:: console
 
-    $ tar xf pypy-2.1.tar.bz2
-    $ ./pypy-2.1/bin/pypy
-    Python 2.7.3 (480845e6b1dd, Jul 31 2013, 11:05:31)
-    [PyPy 2.1.0 with GCC 4.4.3] on linux2
+    $ tar xf pypy-x.y.z.tar.bz2
+    $ ./pypy-x.y.z/bin/pypy
+    Python 2.7.x (xxxxxxxxxxxx, Date, Time)
+    [PyPy x.y.z with GCC x.y.z] on linux2
     Type "help", "copyright", "credits" or "license" for more information.
     And now for something completely different: ``PyPy is an exciting technology
     that lets you to write fast, portable, multi-platform interpreters with less
@@ -57,6 +57,7 @@
 .. code-block:: console
 
     $ ./pypy-xxx/bin/pypy -m ensurepip
+    $ ./pypy-xxx/bin/pip install -U pip wheel # to upgrade to the latest versions
     $ ./pypy-xxx/bin/pip install pygments  # for example
 
 Third party libraries will be installed in ``pypy-xxx/site-packages``, and
@@ -77,7 +78,17 @@
 	# from the mercurial checkout
 	$ virtualenv -p /path/to/pypy/pypy/translator/goal/pypy-c my-pypy-env
 
-Note that bin/python is now a symlink to bin/pypy.
+	# in any case activate it
+	$ source my-pypy-env/bin/activate
+
+Note that my-pypy-env/bin/python is now a symlink to my-pypy-env/bin/pypy
+so you should be able to run pypy simply by typing::
+
+    $ python
+
+You should still upgrade pip and wheel to the latest versions via::
+
+    $ my-pypy-env/bin/pip install -U pip wheel
 
 .. _pip: http://pypi.python.org/pypi/pip
 .. _ensurepip: https://docs.python.org/2.7/library/ensurepip.html
diff --git a/pypy/doc/release-v5.7.0.rst b/pypy/doc/release-v5.7.0.rst
--- a/pypy/doc/release-v5.7.0.rst
+++ b/pypy/doc/release-v5.7.0.rst
@@ -2,8 +2,11 @@
 PyPy2.7 and PyPy3.5 v5.7 - two in one release
 =============================================
 
-We have released PyPy2.7 v5.7, and a beta-quality PyPy3.5 v5.7 (for
-Linux 64bit only at first).
+The PyPy team is proud to release both PyPy2.7 v5.7 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.7 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
 3.5.3.
@@ -88,14 +91,15 @@
 * New features and cleanups
 
   * update the format of the PYPYLOG file and improvements to vmprof
-  * emit more sysconfig values for downstream cextension packages
+  * emit more sysconfig values for downstream cextension packages including
+    properly setting purelib and platlib to site-packages
   * add ``PyAnySet_Check``, ``PyModule_GetName``, ``PyWeakref_Check*``,
     ``_PyImport_{Acquire,Release}Lock``, ``PyGen_Check*``, ``PyOS_AfterFork``,
   * detect and raise on recreation of a PyPy object from a PyObject during
     tp_dealloc
   * refactor and clean up poor handling of unicode exposed in work on py3.5
   * builtin module cppyy_ supports C++ 11, 14, etc. via cling (reflex has been removed)
-  * adapt ``weakref`` according to CPython issue #19542_, will be in CPython 2.7.14
+  * adapt ``weakref`` according to CPython issue 19542_, will be in CPython 2.7.14
   * support translations with cpyext and the Boehm GC (for special cases like
     RevDB_
   * implement ``StringBuffer.get_raw_address`` for the buffer protocol, it is
@@ -121,16 +125,18 @@
   * disable ``clock_gettime()`` on OS/X, since we support 10.11 and it was only
     added in 10.12
   * support ``HAVE_FSTATVFS`` which was unintentionally always false
-  * fix user-created C-API heaptype, issue #2434_
+  * fix user-created C-API heaptype, issue 2434_
   * fix ``PyDict_Update`` is not actually the same as ``dict.update``
   * assign ``tp_doc`` on ``PyTypeObject`` and tie it to the app-level ``__doc__`` attribute
-    issue #2446_
+    issue 2446_
   * clean up memory leaks around ``PyObject_GetBuffer``, ``PyMemoryView_GET_BUFFER``,
     ``PyMemoryView_FromBuffer``, and ``PyBuffer_Release``
   * improve support for creating C-extension objects from app-level classes,
     filling more slots, especially ``tp_new`` and ``tp_dealloc``
-  * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue #2475_
+  * fix for ``ctypes.c_bool`` returning ``bool`` restype, issue 2475_
   * fix in corner cases with the GIL and C-API functions
+  * allow overriding thread.local.__init__ in a subclass, issue 2501_
+  * allow ``PyClass_New`` to be called with NULL as the first arguemnt, issue 2504_
 
 
 * Performance improvements:
@@ -183,21 +189,19 @@
 * Performance improvements:
 
   * do not create a list whenever ``descr_new`` of a ``bytesobject`` is called
-  * 
-  * 
-  * 
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
   * PEP 489: Multi-phase extension module initialization
-  * XXX what else?
 
 .. _resolved: whatsnew-pypy2-5.7.0.html
 .. _19542: https://bugs.python.org/issue19542
 .. _2434: https://bitbucket.org/pypy/pypy/issues/2434/support-pybind11-in-conjunction-with-pypys
 .. _2446: https://bitbucket.org/pypy/pypy/issues/2446/cpyext-tp_doc-field-not-reflected-on
 .. _2475: https://bitbucket.org/pypy/pypy/issues/2475
+.. _2501: https://bitbucket.org/pypy/pypy/issues/2501
+.. _2504: https://bitbucket.org/pypy/pypy/issues/2504
 .. _RevDB: https://bitbucket.org/pypy/revdb
 .. _cryptography: https://cryptography.io
 .. _cppyy: cppyy.html
diff --git a/pypy/doc/release-v5.7.1.rst b/pypy/doc/release-v5.7.1.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.7.1.rst
@@ -0,0 +1,50 @@
+==========
+PyPy 5.7.1
+==========
+
+We have released a bugfix PyPy2.7-v5.7.1 and PyPy3.5-v5.7.1 beta (Linux 64bit),
+due to the following issues:
+
+  * correctly handle an edge case in dict.pop (issue 2508_)
+
+  * fix a regression to correctly handle multiple inheritance in a C-API type
+    where the second base is an app-level class with a ``__new__`` function
+
+  * fix a regression to fill a C-API type's ``tp_getattr`` slot from a
+    ``__getattr__`` method (issue 2523_)
+
+Thanks to those who reported the issues.
+
+.. _2508: https://bitbucket.org/pypy/pypy/issues/2508
+.. _2523: https://bitbucket.org/pypy/pypy/issues/2523
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
+
+The PyPy Team
+
diff --git a/pypy/doc/test/test_whatsnew.py b/pypy/doc/test/test_whatsnew.py
--- a/pypy/doc/test/test_whatsnew.py
+++ b/pypy/doc/test/test_whatsnew.py
@@ -102,6 +102,8 @@
     assert not not_documented
     if branch == 'py3k':
         assert not not_merged
+    else:
+        assert branch in documented, 'Please document this branch before merging: %s' % branch
 
 def test_startrev_on_default():
     doc = ROOT.join('pypy', 'doc')
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -2,7 +2,69 @@
 What's new in PyPy2.7 5.8+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7
+.. this is a revision shortly after release-pypy2.7-v5.7.0
 .. startrev: 44f31f6dd39f
 
+Add cpyext interfaces for ``PyModule_New``
 
+Correctly handle `dict.pop`` where the ``pop``
+key is not the same type as the ``dict``'s and ``pop``
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+.. branch: lstrip_to_empty_string
+
+.. branch: vmprof-native
+
+PyPy support to profile native frames in vmprof.
+
+.. branch: reusing-r11
+.. branch: branch-prediction
+
+Performance tweaks in the x86 JIT-generated machine code: rarely taken
+blocks are moved off-line.  Also, the temporary register used to contain
+large constants is reused across instructions.
+
+.. branch: vmprof-0.4.4
+
+.. branch: controller-refactor
+
+Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -5,3 +5,24 @@
 .. this is the revision after release-pypy3.3-5.7.x was branched
 .. startrev: afbf09453369
 
+.. branch: mtest
+Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+instead of our outdated regrverbose.py script.
+
+.. branch: win32-faulthandler
+
+Enable the 'faulthandler' module on Windows;
+this unblocks the Python test suite.
+
+.. branch: superjumbo
+
+Implement posix.posix_fallocate() and posix.posix_fadvise()
+
+.. branch: py3.5-mac-translate
+
+Fix for different posix primitives on MacOS
+
+.. branch: PyBuffer
+
+Internal refactoring of memoryviews and buffers, fixing some related
+performance issues.
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -11,7 +11,7 @@
 
 To build pypy-c you need a working python environment, and a C compiler.
 It is possible to translate with a CPython 2.6 or later, but this is not
-the preferred way, because it will take a lot longer to run &#8211; depending
+the preferred way, because it will take a lot longer to run &#65533; depending
 on your architecture, between two and three times as long. So head to
 `our downloads`_ and get the latest stable version.
 
@@ -23,11 +23,11 @@
 Installing Visual Compiler v9 (for Python 2.7)
 ----------------------------------------------
 
-This compiler, while the standard one for Python 2.7, is depricated. Microsoft has
+This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has
 made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link
 was checked in Nov 2016). Note that the compiler suite will be installed in
 ``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``.
-Using a current version of ``setuptools`` will be able to find it there. For
+A current version of ``setuptools`` will be able to find it there. For
 Windows 10, you must right-click the download, and under ``Properties`` ->
 ``Compatibility`` mark it as ``Run run this program in comatibility mode for``
 ``Previous version...``. Also, you must download and install the ``.Net Framework 3.5``,
@@ -40,8 +40,9 @@
 Translating PyPy with Visual Studio
 -----------------------------------
 
-We routinely test translation using Visual Studio 2008, Express
-Edition.  Other configurations may work as well.
+We routinely test translation using v9, also known as Visual Studio 2008.
+Our buildbot is still using the Express Edition, not the compiler noted above.
+Other configurations may work as well.
 
 The translation scripts will set up the appropriate environment variables
 for the compiler, so you do not need to run vcvars before translation.
@@ -117,6 +118,9 @@
 -----------------------------------------------------------
 
 Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
+(for post-5.7.1 builds) with sha256 checksum 
+``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
 https://bitbucket.org/pypy/pypy/downloads/local.zip
@@ -124,9 +128,9 @@
 Then expand it into the base directory (base_dir) and modify your environment
 to reflect this::
 
-    set PATH=<base_dir>\bin;<base_dir>\tcltk\bin;%PATH%
-    set INCLUDE=<base_dir>\include;<base_dir>\tcltk\include;%INCLUDE%
-    set LIB=<base_dir>\lib;<base_dir>\tcltk\lib;%LIB%
+    set PATH=<base_dir>\bin;%PATH%
+    set INCLUDE=<base_dir>\include;%INCLUDE%
+    set LIB=<base_dir>\lib;%LIB%
 
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
@@ -173,13 +177,14 @@
 The zlib compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Download http://www.gzip.org/zlib/zlib-1.2.3.tar.gz and extract it in
-the base directory.  Then compile as a static library::
+Download http://www.gzip.org/zlib/zlib-1.2.11.tar.gz and extract it in
+the base directory.  Then compile::
 
-    cd zlib-1.2.3
+    cd zlib-1.2.11
     nmake -f win32\Makefile.msc
     copy zlib.lib <somewhere in LIB>
     copy zlib.h zconf.h <somewhere in INCLUDE>
+    copy zlib1.dll <in PATH> # (needed for tests via ll2ctypes)
 
 
 The bz2 compression library
@@ -198,22 +203,23 @@
 
 PyPy uses cffi to interact with sqlite3.dll. Only the dll is needed, the cffi
 wrapper is compiled when the module is imported for the first time.
-The sqlite3.dll should be version 3.6.21 for CPython2.7 compatablility.
+The sqlite3.dll should be version 3.8.11 for CPython2.7 compatablility.
 
 
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
 Download the source code of expat on sourceforge:
-http://sourceforge.net/projects/expat/ and extract it in the base directory.
-Version 2.1.0 is known to pass tests. Then open the project file ``expat.dsw``
+https://github.com/libexpat/libexpat/releases and extract it in the base directory.
+Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
 with Visual Studio; follow the instruction for converting the project files,
 switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build.

From pypy.commits at gmail.com  Tue May 30 09:53:21 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:21 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: update version to 5.9.0,
 restart whatsnew part 1
Message-ID: <592d7951.b885df0a.cad2b.ed18@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r91453:5f63a4082bb8
Date: 2017-05-30 16:39 +0300
http://bitbucket.org/pypy/pypy/changeset/5f63a4082bb8/

Log:	update version to 5.9.0, restart whatsnew part 1

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
rename from pypy/doc/whatsnew-pypy3-head.rst
rename to pypy/doc/whatsnew-pypy3-5.8.0.rst
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.5.3"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.8.0-alpha0"
-#define PYPY_VERSION_NUM  0x05080000
+#define PYPY_VERSION "5.9.0-alpha0"
+#define PYPY_VERSION_NUM  0x05090000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 8, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 9, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue May 30 09:53:23 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: restart whatsnew part 2
Message-ID: <592d7953.51b21c0a.92602.612c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r91454:c173df164527
Date: 2017-05-30 16:41 +0300
http://bitbucket.org/pypy/pypy/changeset/c173df164527/

Log:	restart whatsnew part 2

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -0,0 +1,28 @@
+=========================
+What's new in PyPy3 5.8+
+=========================
+
+.. this is the revision after release-pypy3.3-5.8.x was branched
+.. startrev: 0f08064cf67c
+
+.. branch: mtest
+Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+instead of our outdated regrverbose.py script.
+
+.. branch: win32-faulthandler
+
+Enable the 'faulthandler' module on Windows;
+this unblocks the Python test suite.
+
+.. branch: superjumbo
+
+Implement posix.posix_fallocate() and posix.posix_fadvise()
+
+.. branch: py3.5-mac-translate
+
+Fix for different posix primitives on MacOS
+
+.. branch: PyBuffer
+
+Internal refactoring of memoryviews and buffers, fixing some related
+performance issues.

From pypy.commits at gmail.com  Tue May 30 09:53:25 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:25 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release
 branch
Message-ID: <592d7955.0ba3df0a.1785b.3498@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91455:a31624746a5a
Date: 2017-05-30 16:43 +0300
http://bitbucket.org/pypy/pypy/changeset/a31624746a5a/

Log:	merge py3.5 into release branch

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
copy from pypy/doc/whatsnew-pypy3-head.rst
copy to pypy/doc/whatsnew-pypy3-5.8.0.rst
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,9 +1,9 @@
 =========================
-What's new in PyPy3 5.7+
+What's new in PyPy3 5.8+
 =========================
 
-.. this is the revision after release-pypy3.3-5.7.x was branched
-.. startrev: afbf09453369
+.. this is the revision after release-pypy3.3-5.8.x was branched
+.. startrev: 0f08064cf67c
 
 .. branch: mtest
 Use "<python> -m test" to run the CPython test suite, as documented by CPython,
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -258,7 +258,7 @@
             lltype.free(oldbuffer, flavor='raw')
 
     def buffer_w(self, space, flags):
-        return ArrayView(ArrayData(self), self.typecode, self.itemsize, False)
+        return ArrayView(ArrayBuffer(self), self.typecode, self.itemsize, False)
 
     def descr_append(self, space, w_x):
         """ append(x)
@@ -848,7 +848,7 @@
     v.typecode = k
 unroll_typecodes = unrolling_iterable(types.keys())
 
-class ArrayData(RawBuffer):
+class ArrayBuffer(RawBuffer):
     _immutable_ = True
     readonly = False
     def __init__(self, w_array):
diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -229,6 +229,14 @@
         'POSIX_FADV_RANDOM', 'POSIX_FADV_NOREUSE', 'POSIX_FADV_DONTNEED']:
             assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
             interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+    
+    if hasattr(rposix, 'sched_get_priority_max'):
+        interpleveldefs['sched_get_priority_max'] = 'interp_posix.sched_get_priority_max'
+        interpleveldefs['sched_get_priority_min'] = 'interp_posix.sched_get_priority_min'
+        for _name in ['SCHED_FIFO', 'SCHED_RR', 'SCHED_OTHER',
+        'SCHED_BATCH']:
+            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
+            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
 
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -2376,6 +2376,10 @@
 
 @unwrap_spec(fd=c_int)
 def get_blocking(space, fd):
+    """get_blocking(fd) -> bool
+
+Get the blocking mode of the file descriptor:
+False if the O_NONBLOCK flag is set, True if the flag is cleared."""
     try:
         flags = rposix.get_status_flags(fd)
     except OSError as e:
@@ -2384,6 +2388,12 @@
 
 @unwrap_spec(fd=c_int, blocking=int)
 def set_blocking(space, fd, blocking):
+    """\
+set_blocking(fd, blocking)
+
+Set the blocking mode of the specified file descriptor.
+Set the O_NONBLOCK flag if blocking is False,
+clear the O_NONBLOCK flag otherwise."""
     try:
         flags = rposix.get_status_flags(fd)
         if blocking:
@@ -2396,6 +2406,10 @@
 
 @unwrap_spec(out=c_int, count=int)
 def sendfile(space, out, w_in, w_offset, count):
+    """\
+sendfile(out, in, offset, count[, headers][, trailers], flags=0)
+            -> byteswritten
+Copy count bytes from file descriptor in to file descriptor out."""
     # why is an argument called "in"???  that doesn't make sense (it is
     # a reserved word), but that's what CPython does
     in_ = space.c_int_w(w_in)
@@ -2418,3 +2432,31 @@
             except OSError as e:
                 wrap_oserror(space, e, eintr_retry=True)
     return space.newint(res)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_max(space, policy):
+    """returns the maximum priority value that 
+    can be used with the scheduling algorithm 
+    identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_max(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
+
+ at unwrap_spec(policy=int)
+def sched_get_priority_min(space, policy):
+    """returns the minimum priority value that
+     can be used with the scheduling algorithm 
+     identified by policy
+    """
+    while True:
+        try:
+            s = rposix.sched_get_priority_min(policy)
+        except OSError as e:
+            wrap_oserror(space, e, eintr_retry=True)
+        else:
+           return space.newint(s)
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -937,6 +937,34 @@
             assert os.WIFEXITED(status1)
             assert os.WEXITSTATUS(status1) == 0   # else, test failure
 
+    if hasattr(rposix, 'sched_get_priority_max'):
+        def test_os_sched_get_priority_max(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_max(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_max(posix.SCHED_BATCH) != -1
+
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_get_priority_min(self):
+            import sys
+            posix, os = self.posix, self.os
+            assert posix.sched_get_priority_min(posix.SCHED_FIFO) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_RR) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_OTHER) != -1
+            assert posix.sched_get_priority_min(posix.SCHED_BATCH) != -1
+            
+    if hasattr(rposix, 'sched_get_priority_min'):
+        def test_os_sched_priority_max_greater_than_min(self):
+            posix, os = self.posix, self.os
+            policy = posix.SCHED_RR
+            low = posix.sched_get_priority_min(policy)
+            high = posix.sched_get_priority_max(policy)
+            assert isinstance(low, int) == True
+            assert isinstance(high, int) == True
+            assert  high > low
+
     def test_write_buffer(self):
         os = self.posix
         fd = os.open(self.path2 + 'test_write_buffer',
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -235,6 +235,7 @@
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'sys/resource.h',
+                'sched.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
@@ -255,6 +256,10 @@
     PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
     PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
     PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
+    SCHED_FIFO = rffi_platform.DefinedConstantInteger('SCHED_FIFO')
+    SCHED_RR = rffi_platform.DefinedConstantInteger('SCHED_RR')
+    SCHED_OTHER = rffi_platform.DefinedConstantInteger('SCHED_OTHER')
+    SCHED_BATCH = rffi_platform.DefinedConstantInteger('SCHED_BATCH')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -1818,6 +1823,21 @@
     def setpriority(which, who, prio):
         handle_posix_error('setpriority', c_setpriority(which, who, prio))
 
+    c_sched_get_priority_max = external('sched_get_priority_max', [rffi.INT],
+                              rffi.INT, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_sched_get_priority_min = external('sched_get_priority_min', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @enforceargs(int)
+    def sched_get_priority_max(policy):
+        return handle_posix_error('sched_get_priority_max', c_sched_get_priority_max(policy))
+
+    @enforceargs(int)
+    def sched_get_priority_min(policy):
+        return handle_posix_error('sched_get_priority_min', c_sched_get_priority_min(policy))
+
+
+
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -781,3 +781,27 @@
             raise
     finally:
         os.close(fd)
+
+ at rposix_requires('sched_get_priority_max')
+def test_sched_get_priority_max():
+    assert rposix.sched_get_priority_max(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_sched_get_priority_min():
+    assert rposix.sched_get_priority_min(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_os_sched_priority_max_greater_than_min():
+    policy = rposix.SCHED_RR
+    low = rposix.sched_get_priority_min(policy)
+    high = rposix.sched_get_priority_max(policy)
+    assert isinstance(low, int) == True
+    assert isinstance(high, int) == True
+    assert  high > low
+

From pypy.commits at gmail.com  Tue May 30 09:53:30 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:30 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release
 branch
Message-ID: <592d795a.d28bdf0a.975d5.c07d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91458:4a11c013124d
Date: 2017-05-30 16:45 +0300
http://bitbucket.org/pypy/pypy/changeset/4a11c013124d/

Log:	merge default into release branch

diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,7 +37,10 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            o = typ()
+            try:
+                o = typ()
+            except NotImplementedError as e:
+                self.skipTest(str(e))    # for PyPy
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/2.7/zipfile.py
@@ -622,19 +622,23 @@
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached..
         """
-        buf = ''
+        # PyPy modification: don't do repeated string concatenation
+        buf = []
+        lenbuf = 0
         if n is None:
             n = -1
         while True:
             if n < 0:
                 data = self.read1(n)
-            elif n > len(buf):
-                data = self.read1(n - len(buf))
+            elif n > lenbuf:
+                data = self.read1(n - lenbuf)
             else:
-                return buf
+                break
             if len(data) == 0:
-                return buf
-            buf += data
+                break
+            lenbuf += len(data)
+            buf.append(data)
+        return "".join(buf)
 
     def _update_crc(self, newdata, eof):
         # Update the CRC using the given data.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -357,6 +357,26 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+C-API Differences
+-----------------
+
+The external C-API has been reimplemented in PyPy as an internal cpyext module.
+We support most of the documented C-API, but sometimes internal C-abstractions
+leak out on CPython and are abused, perhaps even unknowingly. For instance,
+assignment to a ``PyTupleObject`` is not supported after the tuple is
+used internally, even by another C-API function call. On CPython this will
+succeed as long as the refcount is 1.  On PyPy this will always raise a
+``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
+exception (explicitly listed here for search engines).
+
+Another similar problem is assignment of a new function pointer to any of the
+``tp_as_*`` structures after calling ``PyType_Ready``. For instance, overriding
+``tp_as_number.nb_int`` with a different function after calling ``PyType_Ready``
+on CPython will result in the old function being called for ``x.__int__()``
+(via class ``__dict__`` lookup) and the new function being called for ``int(x)``
+(via slot lookup). On PyPy we will always call the __new__ function, not the
+old, this quirky behaviour is unfortunately necessary to fully support NumPy.
+
 Performance Differences
 -------------------------
 
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
    release-pypy2.7-v5.6.0.rst
@@ -60,6 +61,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.8.0.rst
@@ -0,0 +1,131 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.8 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
+3.5.3.
+
+We continue to make incremental improvements to our C-API
+compatibility layer (cpyext). PyPy2 can now import and run many C-extension
+packages, among the most notable are Numpy, Cython, and Pandas. Performance may
+be slower than CPython, especially for frequently-called short C functions.
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
+as `these benchmarks show`_ it already gives a nice speed bump.
+We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
+"PyPy3.5" supports the Python 3.5 language).
+
+CFFI_ has been updated to 1.10, improving an already great package for
+interfacing with C.
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.8 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.7 released March, 2017)
+=======================================================================================
+
+See also issues that were resolved_
+
+* New features and cleanups
+
+  * 
+  * 
+
+* Bug Fixes
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+  * 
+
+* RPython improvements
+
+  * 
+  * 
+
+
+Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
+======================================================================
+
+* New features
+
+  * 
+  * 
+
+* Bug Fixes
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+  * PEP 489: Multi-phase extension module initialization
+
+.. _resolved: whatsnew-pypy2-5.8.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,70 +1,8 @@
 ==========================
-What's new in PyPy2.7 5.8+
+What's new in PyPy2.7 5.9+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7.0
-.. startrev: 44f31f6dd39f
+.. this is a revision shortly after release-pypy2.7-v5.8.0
+.. startrev: 0c91b5f4f275
 
-Add cpyext interfaces for ``PyModule_New``
 
-Correctly handle `dict.pop`` where the ``pop``
-key is not the same type as the ``dict``'s and ``pop``
-is called with a default (will be part of release 5.7.1)
-
-.. branch: issue2522
-
-Fix missing tp_new on w_object called through multiple inheritance
-(will be part of release 5.7.1)
-
-.. branch: lstrip_to_empty_string
-
-.. branch: vmprof-native
-
-PyPy support to profile native frames in vmprof.
-
-.. branch: reusing-r11
-.. branch: branch-prediction
-
-Performance tweaks in the x86 JIT-generated machine code: rarely taken
-blocks are moved off-line.  Also, the temporary register used to contain
-large constants is reused across instructions.
-
-.. branch: vmprof-0.4.4
-
-.. branch: controller-refactor
-
-Refactor rpython.rtyper.controllerentry.
-
-.. branch: PyBuffer-backport
-
-Internal refactoring of buffers and memoryviews. Memoryviews will now be
-accepted in a few more places, e.g. in compile().
-
-.. branch: sthalik/fix-signed-integer-sizes-1494493539409
-
-.. branch: cpyext-obj-stealing
-
-Redo much of the refcount semantics in PyList_{SG}etItem to closer match
-CPython and ensure the same PyObject stored in the list can be later
-retrieved
-
-.. branch: cpyext-recursionlimit
-
-Implement Py_EnterRecursiveCall and associated functions
-
-.. branch: pypy_ctypes_nosegfault_nofastpath
-
-Remove faulty fastpath from ctypes
-
-.. branch: sockopt_zero
-
-Passing a buffersize of 0 to socket.getsockopt
-
-.. branch: better-test-whatsnew
-
-.. branch: faster-rstruct-2
-
-Improve the performance of struct.pack and struct.pack_into by using raw_store
-or gc_store_indexed whenever possible. Moreover, enable the existing
-struct.unpack fast path to all the existing buffer types, whereas previously
-it was enabled only for strings
diff --git a/pypy/doc/whatsnew-pypy2-5.8.0.rst b/pypy/doc/whatsnew-pypy2-5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.8.0.rst
@@ -0,0 +1,77 @@
+==========================
+What's new in PyPy2.7 5.8+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.7.0
+.. startrev: 44f31f6dd39f
+
+Add cpyext interfaces for ``PyModule_New``
+
+Correctly handle `dict.pop`` where the ``pop``
+key is not the same type as the ``dict``'s and ``pop``
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+.. branch: lstrip_to_empty_string
+
+.. branch: vmprof-native
+
+PyPy support to profile native frames in vmprof.
+
+.. branch: reusing-r11
+.. branch: branch-prediction
+
+Performance tweaks in the x86 JIT-generated machine code: rarely taken
+blocks are moved off-line.  Also, the temporary register used to contain
+large constants is reused across instructions.
+
+.. branch: vmprof-0.4.4
+
+.. branch: controller-refactor
+
+Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings
+
+.. branch: Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235
+
+.. branch: PyPy_profopt_enabled
+
+Add profile-based optimization option ``profopt``, and specify training data
+via ``profoptpath``
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -798,8 +798,8 @@
                               '"license" for more information.')
 
 STDLIB_WARNING = """\
-debug: WARNING: Library path not found, using compiled-in sys.path.
-debug: WARNING: 'sys.prefix' will not be set.
+debug: WARNING: Library path not found, using compiled-in sys.path, with
+debug: WARNING: sys.prefix = %r
 debug: WARNING: Make sure the pypy binary is kept inside its tree of files.
 debug: WARNING: It is ok to create a symlink to it from somewhere else."""
 
@@ -818,12 +818,8 @@
     executable = sys.pypy_find_executable(executable)
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
-        for lib_path in sys.path:
-            stdlib_path = sys.pypy_find_stdlib(lib_path)
-            if stdlib_path is not None:
-                break
-    if stdlib_path is None:
-        print >> sys.stderr, STDLIB_WARNING
+        print >> sys.stderr, STDLIB_WARNING % (
+            getattr(sys, 'prefix', '<missing>'),)
     else:
         sys.path[:] = stdlib_path
     # from this point on, we are free to use all the unicode stuff we want,
diff --git a/pypy/module/__builtin__/operation.py b/pypy/module/__builtin__/operation.py
--- a/pypy/module/__builtin__/operation.py
+++ b/pypy/module/__builtin__/operation.py
@@ -6,7 +6,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
 from rpython.rlib.runicode import UNICHR
-from rpython.rlib.rfloat import isnan, isinf, round_double
+from rpython.rlib.rfloat import isfinite, isinf, round_double, round_away
 from rpython.rlib import rfloat
 import __builtin__
 
@@ -134,23 +134,26 @@
     ndigits = space.getindex_w(w_ndigits, None)
 
     # nans, infinities and zeros round to themselves
-    if number == 0 or isinf(number) or isnan(number):
-        return space.newfloat(number)
-
-    # Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
-    # always rounds to itself.  For ndigits < NDIGITS_MIN, x always
-    # rounds to +-0.0.
-    if ndigits > NDIGITS_MAX:
-        return space.newfloat(number)
-    elif ndigits < NDIGITS_MIN:
-        # return 0.0, but with sign of x
-        return space.newfloat(0.0 * number)
-
-    # finite x, and ndigits is not unreasonably large
-    z = round_double(number, ndigits)
-    if isinf(z):
-        raise oefmt(space.w_OverflowError,
-                    "rounded value too large to represent")
+    if not isfinite(number):
+        z = number
+    elif ndigits == 0:    # common case
+        z = round_away(number)
+        # no need to check for an infinite 'z' here
+    else:
+        # Deal with extreme values for ndigits. For ndigits > NDIGITS_MAX, x
+        # always rounds to itself.  For ndigits < NDIGITS_MIN, x always
+        # rounds to +-0.0.
+        if ndigits > NDIGITS_MAX:
+            z = number
+        elif ndigits < NDIGITS_MIN:
+            # return 0.0, but with sign of x
+            z = 0.0 * number
+        else:
+            # finite x, and ndigits is not unreasonably large
+            z = round_double(number, ndigits)
+            if isinf(z):
+                raise oefmt(space.w_OverflowError,
+                            "rounded value too large to represent")
     return space.newfloat(z)
 
 # ____________________________________________________________
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -625,6 +625,9 @@
         assert round(5e15) == 5e15
         assert round(-(5e15-1)) == -(5e15-1)
         assert round(-5e15) == -5e15
+        assert round(5e15/2) == 5e15/2
+        assert round((5e15+1)/2) == 5e15/2+1
+        assert round((5e15-1)/2) == 5e15/2
         #
         inf = 1e200 * 1e200
         assert round(inf) == inf
@@ -636,6 +639,12 @@
         #
         assert round(562949953421312.5, 1) == 562949953421312.5
         assert round(56294995342131.5, 3) == 56294995342131.5
+        #
+        for i in range(-10, 10):
+            expected = i if i < 0 else i + 1
+            assert round(i + 0.5) == round(i + 0.5, 0) == expected
+            x = i * 10 + 5
+            assert round(x, -1) == round(float(x), -1) == expected * 10
 
     def test_vars_obscure_case(self):
         class C_get_vars(object):
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -310,6 +310,8 @@
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
+        if buflen < 0 or buflen > 1024:
+            raise explicit_socket_error(space, "getsockopt buflen out of range")
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
@@ -696,6 +698,12 @@
         w_exception = space.call_function(w_exception_class, space.newtext(message))
     return OperationError(w_exception_class, w_exception)
 
+def explicit_socket_error(space, msg):
+    w_exception_class = space.fromcache(SocketAPI).w_error
+    w_exception = space.call_function(w_exception_class, space.newtext(msg))
+    return OperationError(w_exception_class, w_exception)
+
+
 # ____________________________________________________________
 
 socketmethodnames = """
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -595,6 +595,16 @@
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
 
+    def test_getsockopt_bad_length(self):
+        import _socket
+        s = _socket.socket()
+        buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
+        assert buf == b'\x00' * 4
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, -1)
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':
diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -799,51 +799,39 @@
 class ArrayBuffer(RawBuffer):
     _immutable_ = True
 
-    def __init__(self, array, readonly):
-        self.array = array
+    def __init__(self, w_array, readonly):
+        self.w_array = w_array
         self.readonly = readonly
 
     def getlength(self):
-        return self.array.len * self.array.itemsize
-
-    def getformat(self):
-        return self.array.typecode
-
-    def getitemsize(self):
-        return self.array.itemsize
-
-    def getndim(self):
-        return 1
-
-    def getstrides(self):
-        return [self.getitemsize()]
+        return self.w_array.len * self.w_array.itemsize
 
     def getitem(self, index):
-        array = self.array
-        data = array._charbuf_start()
+        w_array = self.w_array
+        data = w_array._charbuf_start()
         char = data[index]
-        array._charbuf_stop()
+        w_array._charbuf_stop()
         return char
 
     def setitem(self, index, char):
-        array = self.array
-        data = array._charbuf_start()
+        w_array = self.w_array
+        data = w_array._charbuf_start()
         data[index] = char
-        array._charbuf_stop()
+        w_array._charbuf_stop()
 
     def getslice(self, start, stop, step, size):
         if size == 0:
             return ''
         if step == 1:
-            data = self.array._charbuf_start()
+            data = self.w_array._charbuf_start()
             try:
                 return rffi.charpsize2str(rffi.ptradd(data, start), size)
             finally:
-                self.array._charbuf_stop()
+                self.w_array._charbuf_stop()
         return RawBuffer.getslice(self, start, stop, step, size)
 
     def get_raw_address(self):
-        return self.array._charbuf_start()
+        return self.w_array._charbuf_start()
 
 
 unpack_driver = jit.JitDriver(name='unpack_array',
diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py
--- a/pypy/module/cpyext/bufferobject.py
+++ b/pypy/module/cpyext/bufferobject.py
@@ -56,9 +56,9 @@
         py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, rffi.str2charp(buf.value))
         py_buf.c_b_size = buf.getlength()
     elif isinstance(buf, ArrayBuffer):
-        w_base = buf.array
+        w_base = buf.w_array
         py_buf.c_b_base = make_ref(space, w_base)
-        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.array._charbuf_start())
+        py_buf.c_b_ptr = rffi.cast(rffi.VOIDP, buf.w_array._charbuf_start())
         py_buf.c_b_size = buf.getlength()
     else:
         raise oefmt(space.w_NotImplementedError, "buffer flavor not supported")
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -161,3 +161,47 @@
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
         assert list(a) == range(100, 400, 100)
+
+    def test_setitem(self):
+        module = self.import_extension('foo', [
+            ("set_after_use", "METH_O",
+             """
+                PyObject *t2, *tuple = PyTuple_New(1);
+                PyObject * one = PyLong_FromLong(1);
+                int res;
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                Py_INCREF(args);
+                res = PyTuple_SetItem(tuple, 0, args);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                /* Do something that uses the tuple, but does not incref */
+                t2 = PyTuple_GetSlice(tuple, 0, 1);
+                Py_DECREF(t2);
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                Py_DECREF(tuple);
+                if (res != 0)
+                {
+                    Py_DECREF(one);
+                    return NULL;
+                }
+                Py_INCREF(Py_None);
+                return Py_None;
+             """),
+            ])
+        import sys
+        s = 'abc'
+        if '__pypy__' in sys.builtin_module_names:
+            raises(SystemError, module.set_after_use, s)
+        else:
+            module.set_after_use(s)
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -6,7 +6,7 @@
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
-    track_reference, make_typedescr, get_typedescr)
+    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -132,19 +132,20 @@
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyTuple_SetItem(space, ref, index, py_obj):
-    # XXX this will not complain when changing tuples that have
-    # already been realized as a W_TupleObject, but won't update the
-    # W_TupleObject
     if not tuple_check_ref(space, ref):
         decref(space, py_obj)
         PyErr_BadInternalCall(space)
-    ref = rffi.cast(PyTupleObject, ref)
-    size = ref.c_ob_size
+    tupleobj = rffi.cast(PyTupleObject, ref)
+    size = tupleobj.c_ob_size
     if index < 0 or index >= size:
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
-    old_ref = ref.c_ob_item[index]
-    ref.c_ob_item[index] = py_obj    # consumes a reference
+    old_ref = tupleobj.c_ob_item[index]
+    if pyobj_has_w_obj(ref):
+        # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
+        raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
+                                        " use of tuple")
+    tupleobj.c_ob_item[index] = py_obj    # consumes a reference
     if old_ref:
         decref(space, old_ref)
     return 0
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -39,8 +39,8 @@
         'stderr'                : 'state.getio(space).w_stderr',
         '__stderr__'            : 'state.getio(space).w_stderr',
         'pypy_objspaceclass'    : 'space.newtext(repr(space))',
-        #'prefix'               : # added by pypy_initial_path() when it
-        #'exec_prefix'          : # succeeds, pointing to trunk or /usr
+        'prefix'                : 'state.get(space).w_initial_prefix',
+        'exec_prefix'           : 'state.get(space).w_initial_prefix',
         'path'                  : 'state.get(space).w_path',
         'modules'               : 'state.get(space).w_modules',
         'argv'                  : 'state.get(space).w_argv',
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -19,8 +19,11 @@
 
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
+        # This initial value for sys.prefix is normally overwritten
+        # at runtime by initpath.py
+        srcdir = os.path.dirname(pypydir)
+        self.w_initial_prefix = space.newtext(srcdir)
         # Initialize the default path
-        srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.newtext(p) for p in path])
 
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -648,7 +648,7 @@
         if space.isinstance_w(w_other, space.w_unicode):
             self_as_unicode = unicode_from_encoded_object(space, self, None,
                                                           None)
-            return space.add(self_as_unicode, w_other)
+            return self_as_unicode.descr_add(space, w_other)
         elif space.isinstance_w(w_other, space.w_bytearray):
             # XXX: eliminate double-copy
             from .bytearrayobject import W_BytearrayObject, _make_data
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -846,3 +846,7 @@
                 return 42
         x = Foo()
         assert "hello" + x == 42
+
+    def test_add(self):
+        assert 'abc' + 'abc' == 'abcabc'
+        assert isinstance('abc' + u'\u03a3', unicode)
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -463,6 +463,21 @@
     def get_root_stack_top_addr(self):
         return rffi.cast(lltype.Signed, self.stack_addr)
 
+    def getlength(self):
+        top = self.stack_addr[0]
+        base = rffi.cast(lltype.Signed, self.stack)
+        n = (top - base) // WORD
+        assert 0 <= n < 10
+        return n
+
+    def curtop(self):
+        n = self.getlength()
+        return self.stack[n - 1]
+
+    def settop(self, newvalue):
+        n = self.getlength()
+        self.stack[n - 1] = newvalue
+
 class WriteBarrierDescr(AbstractDescr):
     jit_wb_cards_set = 0
     jit_wb_if_flag_singlebyte = 1
@@ -645,7 +660,7 @@
         frames = []
 
         def check(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert len(frame.jf_frame) == self.cpu.JITFRAME_FIXED_SIZE + 4
             # we "collect"
@@ -665,14 +680,14 @@
                 assert gcmap == [22, 23, 24]
             for item, s in zip(gcmap, new_items):
                 new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s)
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame)
-            cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame)
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == rffi.cast(lltype.Signed, frame)
+            cpu.gc_ll_descr.gcrootmap.settop(rffi.cast(lltype.Signed, new_frame))
             print '"Collecting" moved the frame from %d to %d' % (
-                i, cpu.gc_ll_descr.gcrootmap.stack[0])
+                i, cpu.gc_ll_descr.gcrootmap.curtop())
             frames.append(new_frame)
 
         def check2(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert frame == frames[1]
             assert frame != frames[0]
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1052,17 +1052,19 @@
 
     def _call_header_shadowstack(self, gcrootmap):
         rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap)
-        self.mc.MOV_mr((ebx.value, 0), ebp.value)      # MOV [ebx], ebp
-        self.mc.ADD_ri(ebx.value, WORD)
+        # the '1' is to benefit from the shadowstack 'is_minor' optimization
+        self.mc.MOV_mi((ebx.value, 0), 1)               # MOV [ebx], 1
+        self.mc.MOV_mr((ebx.value, WORD), ebp.value)    # MOV [ebx + WORD], ebp
+        self.mc.ADD_ri(ebx.value, WORD * 2)
         self.mc.MOV(heap(rst), ebx)                   # MOV [rootstacktop], ebx
 
     def _call_footer_shadowstack(self, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
         if rx86.fits_in_32bits(rst):
-            self.mc.SUB_ji8(rst, WORD)       # SUB [rootstacktop], WORD
+            self.mc.SUB_ji8(rst, WORD * 2)       # SUB [rootstacktop], WORD * 2
         else:
             self.mc.MOV_ri(ebx.value, rst)           # MOV ebx, rootstacktop
-            self.mc.SUB_mi8((ebx.value, 0), WORD)  # SUB [ebx], WORD
+            self.mc.SUB_mi8((ebx.value, 0), WORD * 2)  # SUB [ebx], WORD * 2
 
     def redirect_call_assembler(self, oldlooptoken, newlooptoken):
         # some minimal sanity checking
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -925,6 +925,10 @@
     def gct_gc_adr_of_root_stack_top(self, hop):
         self._gc_adr_of_gcdata_attr(hop, 'root_stack_top')
 
+    def gct_gc_modified_shadowstack(self, hop):
+        # for stacklet
+        hop.genop("direct_call", [self.root_walker.gc_modified_shadowstack_ptr])
+
     def gct_gc_detach_callback_pieces(self, hop):
         op = hop.spaceop
         assert len(op.args) == 0
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -245,6 +245,13 @@
         from rpython.rlib import _stacklet_shadowstack
         _stacklet_shadowstack.complete_destrptr(gctransformer)
 
+        gcdata = self.gcdata
+        def gc_modified_shadowstack():
+            gcdata.can_look_at_partial_stack = False
+
+        self.gc_modified_shadowstack_ptr = getfn(gc_modified_shadowstack,
+                                                 [], annmodel.s_None)
+
     def postprocess_graph(self, gct, graph, any_inlining):
         from rpython.memory.gctransform import shadowcolor
         if any_inlining:
diff --git a/rpython/rlib/_stacklet_shadowstack.py b/rpython/rlib/_stacklet_shadowstack.py
--- a/rpython/rlib/_stacklet_shadowstack.py
+++ b/rpython/rlib/_stacklet_shadowstack.py
@@ -77,6 +77,7 @@
     llmemory.raw_memcopy(sscopy + SIZEADDR, base, length_bytes)
     llop.gc_adr_of_root_stack_top(llmemory.Address).address[0] = (
         base + length_bytes)
+    llop.gc_modified_shadowstack(lltype.Void)
     llmemory.raw_free(sscopy)
 
 def alloc_stacklet():
diff --git a/rpython/rlib/rfloat.py b/rpython/rlib/rfloat.py
--- a/rpython/rlib/rfloat.py
+++ b/rpython/rlib/rfloat.py
@@ -96,7 +96,20 @@
     """Round a float half away from zero.
 
     Specify half_even=True to round half even instead.
+    The argument 'value' must be a finite number.  This
+    function may return an infinite number in case of
+    overflow (only if ndigits is a very negative integer).
     """
+    if ndigits == 0:
+        # fast path for this common case
+        if half_even:
+            return round_half_even(value)
+        else:
+            return round_away(value)
+
+    if value == 0.0:
+        return 0.0
+
     # The basic idea is very simple: convert and round the double to
     # a decimal string using _Py_dg_dtoa, then convert that decimal
     # string back to a double with _Py_dg_strtod.  There's one minor
@@ -217,11 +230,34 @@
 
 def round_away(x):
     # round() from libm, which is not available on all platforms!
+    # This version rounds away from zero.
     absx = abs(x)
-    if absx - math.floor(absx) >= .5:
-        r = math.ceil(absx)
+    r = math.floor(absx + 0.5)
+    if r - absx < 1.0:
+        return copysign(r, x)
     else:
-        r = math.floor(absx)
+        # 'absx' is just in the wrong range: its exponent is precisely
+        # the one for which all integers are representable but not any
+        # half-integer.  It means that 'absx + 0.5' computes equal to
+        # 'absx + 1.0', which is not equal to 'absx'.  So 'r - absx'
+        # computes equal to 1.0.  In this situation, we can't return
+        # 'r' because 'absx' was already an integer but 'r' is the next
+        # integer!  But just returning the original 'x' is fine.
+        return x
+
+def round_half_even(x):
+    absx = abs(x)
+    r = math.floor(absx + 0.5)
+    frac = r - absx
+    if frac >= 0.5:
+        # two rare cases: either 'absx' is precisely half-way between
+        # two integers (frac == 0.5); or we're in the same situation as
+        # described in round_away above (frac == 1.0).
+        if frac >= 1.0:
+            return x
+        # absx == n + 0.5  for a non-negative integer 'n'
+        # absx * 0.5 == n//2 + 0.25 or 0.75, which we round to nearest
+        r = math.floor(absx * 0.5 + 0.5) * 2.0
     return copysign(r, x)
 
 @not_rpython
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -235,6 +235,7 @@
     includes = ['unistd.h',  'sys/types.h', 'sys/wait.h',
                 'utime.h', 'sys/time.h', 'sys/times.h',
                 'sys/resource.h',
+                'sched.h',
                 'grp.h', 'dirent.h', 'sys/stat.h', 'fcntl.h',
                 'signal.h', 'sys/utsname.h', _ptyh]
     if sys.platform.startswith('linux'):
@@ -255,6 +256,10 @@
     PRIO_PROCESS = rffi_platform.DefinedConstantInteger('PRIO_PROCESS')
     PRIO_PGRP = rffi_platform.DefinedConstantInteger('PRIO_PGRP')
     PRIO_USER = rffi_platform.DefinedConstantInteger('PRIO_USER')
+    SCHED_FIFO = rffi_platform.DefinedConstantInteger('SCHED_FIFO')
+    SCHED_RR = rffi_platform.DefinedConstantInteger('SCHED_RR')
+    SCHED_OTHER = rffi_platform.DefinedConstantInteger('SCHED_OTHER')
+    SCHED_BATCH = rffi_platform.DefinedConstantInteger('SCHED_BATCH')
     O_NONBLOCK = rffi_platform.DefinedConstantInteger('O_NONBLOCK')
     OFF_T = rffi_platform.SimpleType('off_t')
     OFF_T_SIZE = rffi_platform.SizeOf('off_t')
@@ -1818,6 +1823,21 @@
     def setpriority(which, who, prio):
         handle_posix_error('setpriority', c_setpriority(which, who, prio))
 
+    c_sched_get_priority_max = external('sched_get_priority_max', [rffi.INT],
+                              rffi.INT, save_err=rffi.RFFI_FULL_ERRNO_ZERO)
+    c_sched_get_priority_min = external('sched_get_priority_min', [rffi.INT],
+                             rffi.INT, save_err=rffi.RFFI_SAVE_ERRNO)
+
+    @enforceargs(int)
+    def sched_get_priority_max(policy):
+        return handle_posix_error('sched_get_priority_max', c_sched_get_priority_max(policy))
+
+    @enforceargs(int)
+    def sched_get_priority_min(policy):
+        return handle_posix_error('sched_get_priority_min', c_sched_get_priority_min(policy))
+
+
+
 
 #___________________________________________________________________
 
diff --git a/rpython/rlib/test/test_rfloat.py b/rpython/rlib/test/test_rfloat.py
--- a/rpython/rlib/test/test_rfloat.py
+++ b/rpython/rlib/test/test_rfloat.py
@@ -28,7 +28,7 @@
 
 def test_round_double():
     def almost_equal(x, y):
-        assert round(abs(x-y), 7) == 0
+        assert abs(x-y) < 1e-7
 
     almost_equal(round_double(0.125, 2), 0.13)
     almost_equal(round_double(0.375, 2), 0.38)
@@ -44,35 +44,35 @@
     almost_equal(round_double(-0.25, 1), -0.3)
     almost_equal(round_double(-0.75, 1), -0.8)
 
-    round_double(-6.5, 0) == -7.0
-    round_double(-5.5, 0) == -6.0
-    round_double(-1.5, 0) == -2.0
-    round_double(-0.5, 0) == -1.0
-    round_double(0.5, 0) == 1.0
-    round_double(1.5, 0) == 2.0
-    round_double(2.5, 0) == 3.0
-    round_double(3.5, 0) == 4.0
-    round_double(4.5, 0) == 5.0
-    round_double(5.5, 0) == 6.0
-    round_double(6.5, 0) == 7.0
+    assert round_double(-6.5, 0) == -7.0
+    assert round_double(-5.5, 0) == -6.0
+    assert round_double(-1.5, 0) == -2.0
+    assert round_double(-0.5, 0) == -1.0
+    assert round_double(0.5, 0) == 1.0
+    assert round_double(1.5, 0) == 2.0
+    assert round_double(2.5, 0) == 3.0
+    assert round_double(3.5, 0) == 4.0
+    assert round_double(4.5, 0) == 5.0
+    assert round_double(5.5, 0) == 6.0
+    assert round_double(6.5, 0) == 7.0
 
-    round_double(-25.0, -1) == -30.0
-    round_double(-15.0, -1) == -20.0
-    round_double(-5.0, -1) == -10.0
-    round_double(5.0, -1) == 10.0
-    round_double(15.0, -1) == 20.0
-    round_double(25.0, -1) == 30.0
-    round_double(35.0, -1) == 40.0
-    round_double(45.0, -1) == 50.0
-    round_double(55.0, -1) == 60.0
-    round_double(65.0, -1) == 70.0
-    round_double(75.0, -1) == 80.0
-    round_double(85.0, -1) == 90.0
-    round_double(95.0, -1) == 100.0
-    round_double(12325.0, -1) == 12330.0
+    assert round_double(-25.0, -1) == -30.0
+    assert round_double(-15.0, -1) == -20.0
+    assert round_double(-5.0, -1) == -10.0
+    assert round_double(5.0, -1) == 10.0
+    assert round_double(15.0, -1) == 20.0
+    assert round_double(25.0, -1) == 30.0
+    assert round_double(35.0, -1) == 40.0
+    assert round_double(45.0, -1) == 50.0
+    assert round_double(55.0, -1) == 60.0
+    assert round_double(65.0, -1) == 70.0
+    assert round_double(75.0, -1) == 80.0
+    assert round_double(85.0, -1) == 90.0
+    assert round_double(95.0, -1) == 100.0
+    assert round_double(12325.0, -1) == 12330.0
 
-    round_double(350.0, -2) == 400.0
-    round_double(450.0, -2) == 500.0
+    assert round_double(350.0, -2) == 400.0
+    assert round_double(450.0, -2) == 500.0
 
     almost_equal(round_double(0.5e21, -21), 1e21)
     almost_equal(round_double(1.5e21, -21), 2e21)
@@ -85,6 +85,13 @@
     almost_equal(round_double(0.5e22, -22), 1e22)
     almost_equal(round_double(1.5e22, -22), 2e22)
 
+    exact_integral = 5e15 + 1
+    assert round_double(exact_integral, 0) == exact_integral
+    assert round_double(exact_integral/2.0, 0) == 5e15/2.0 + 1.0
+    exact_integral = 5e15 - 1
+    assert round_double(exact_integral, 0) == exact_integral
+    assert round_double(exact_integral/2.0, 0) == 5e15/2.0
+
 def test_round_half_even():
     from rpython.rlib import rfloat
     func = rfloat.round_double
@@ -92,6 +99,15 @@
     assert func(2.5, 0, False) == 3.0
     # 3.x behavior
     assert func(2.5, 0, True) == 2.0
+    for i in range(-10, 10):
+        assert func(i + 0.5, 0, True) == i + (i & 1)
+        assert func(i * 10 + 5, -1, True) == (i + (i & 1)) * 10
+    exact_integral = 5e15 + 1
+    assert round_double(exact_integral, 0, True) == exact_integral
+    assert round_double(exact_integral/2.0, 0, True) == 5e15/2.0
+    exact_integral = 5e15 - 1
+    assert round_double(exact_integral, 0, True) == exact_integral
+    assert round_double(exact_integral/2.0, 0, True) == 5e15/2.0
 
 def test_float_as_rbigint_ratio():
     for f, ratio in [
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -781,3 +781,27 @@
             raise
     finally:
         os.close(fd)
+
+ at rposix_requires('sched_get_priority_max')
+def test_sched_get_priority_max():
+    assert rposix.sched_get_priority_max(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_max(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_sched_get_priority_min():
+    assert rposix.sched_get_priority_min(rposix.SCHED_FIFO) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_RR) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_OTHER) != -1
+    assert rposix.sched_get_priority_min(rposix.SCHED_BATCH) != -1
+    
+ at rposix_requires('sched_get_priority_min')
+def test_os_sched_priority_max_greater_than_min():
+    policy = rposix.SCHED_RR
+    low = rposix.sched_get_priority_min(policy)
+    high = rposix.sched_get_priority_max(policy)
+    assert isinstance(low, int) == True
+    assert isinstance(high, int) == True
+    assert  high > low
+
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -862,6 +862,9 @@
     def op_gc_adr_of_root_stack_top(self):
         raise NotImplementedError
 
+    def op_gc_modified_shadowstack(self):
+        raise NotImplementedError
+
     def op_gc_call_rtti_destructor(self, rtti, addr):
         if hasattr(rtti._obj, 'destructor_funcptr'):
             d = rtti._obj.destructor_funcptr
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -515,6 +515,7 @@
     'gc_adr_of_root_stack_base': LLOp(),
     'gc_adr_of_root_stack_top': LLOp(),
     # returns the address of gcdata.root_stack_base/top (for shadowstack only)
+    'gc_modified_shadowstack': LLOp(),
 
     # for asmgcroot support to get the address of various static structures
     # see translator/c/src/mem.h for the valid indices
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -102,7 +102,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         raise NotImplementedError("Pure abstract baseclass")
 
     def __repr__(self):
diff --git a/rpython/translator/platform/darwin.py b/rpython/translator/platform/darwin.py
--- a/rpython/translator/platform/darwin.py
+++ b/rpython/translator/platform/darwin.py
@@ -96,7 +96,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         # ensure frameworks are passed in the Makefile
         fs = self._frameworks(eci.frameworks)
         if len(fs) > 0:
@@ -106,7 +106,7 @@
                                 shared=shared,
                                 headers_to_precompile=headers_to_precompile,
                                 no_precompile_cfiles = no_precompile_cfiles,
-                                config=config)
+                                profopt=profopt, config=config)
         return mk
 
 class Darwin_PowerPC(Darwin):#xxx fixme, mwp
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -299,7 +299,7 @@
 
     def gen_makefile(self, cfiles, eci, exe_name=None, path=None,
                      shared=False, headers_to_precompile=[],
-                     no_precompile_cfiles = [], config=None):
+                     no_precompile_cfiles = [], profopt=False, config=None):
         cfiles = self._all_cfiles(cfiles, eci)
 
         if path is None:

From pypy.commits at gmail.com  Tue May 30 09:53:26 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: update development version to 5.9,
 restart whatsnew (part 1)
Message-ID: <592d7956.478fdf0a.d2a4f.fa15@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91456:3169bec9ae7c
Date: 2017-05-30 16:22 +0300
http://bitbucket.org/pypy/pypy/changeset/3169bec9ae7c/

Log:	update development version to 5.9, restart whatsnew (part 1)

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
    release-pypy2.7-v5.6.0.rst
@@ -60,6 +61,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.8.0.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-pypy2-5.8.0.rst
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.13"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.8.0-alpha0"
-#define PYPY_VERSION_NUM  0x05080000
+#define PYPY_VERSION "5.9.0-alpha0"
+#define PYPY_VERSION_NUM  0x05090000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 8, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 9, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Tue May 30 09:53:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:32 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fixes for whatsnew, document merged branch
Message-ID: <592d795c.d2071c0a.11635.bfac@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r91459:d61ec1c88982
Date: 2017-05-30 16:50 +0300
http://bitbucket.org/pypy/pypy/changeset/d61ec1c88982/

Log:	fixes for whatsnew, document merged branch

diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
--- a/pypy/doc/whatsnew-pypy3-5.8.0.rst
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -26,3 +26,7 @@
 
 Internal refactoring of memoryviews and buffers, fixing some related
 performance issues.
+
+.. branch: jumbojet
+
+Add sched_get min/max to rposix
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -3,26 +3,6 @@
 =========================
 
 .. this is the revision after release-pypy3.3-5.8.x was branched
-.. startrev: 0f08064cf67c
+.. startrev: c173df164527
 
-.. branch: mtest
-Use "<python> -m test" to run the CPython test suite, as documented by CPython,
-instead of our outdated regrverbose.py script.
 
-.. branch: win32-faulthandler
-
-Enable the 'faulthandler' module on Windows;
-this unblocks the Python test suite.
-
-.. branch: superjumbo
-
-Implement posix.posix_fallocate() and posix.posix_fadvise()
-
-.. branch: py3.5-mac-translate
-
-Fix for different posix primitives on MacOS
-
-.. branch: PyBuffer
-
-Internal refactoring of memoryviews and buffers, fixing some related
-performance issues.

From pypy.commits at gmail.com  Tue May 30 09:53:28 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:28 -0700 (PDT)
Subject: [pypy-commit] pypy default: start very rough draft of release
 notice, restart whatsnew (part 2)
Message-ID: <592d7958.505c1c0a.edbbb.0eba@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91457:558bd00b3dd8
Date: 2017-05-30 16:31 +0300
http://bitbucket.org/pypy/pypy/changeset/558bd00b3dd8/

Log:	start very rough draft of release notice, restart whatsnew (part 2)

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.8.0.rst
@@ -0,0 +1,131 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.8 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
+3.5.3.
+
+We continue to make incremental improvements to our C-API
+compatibility layer (cpyext). PyPy2 can now import and run many C-extension
+packages, among the most notable are Numpy, Cython, and Pandas. Performance may
+be slower than CPython, especially for frequently-called short C functions.
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
+as `these benchmarks show`_ it already gives a nice speed bump.
+We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
+"PyPy3.5" supports the Python 3.5 language).
+
+CFFI_ has been updated to 1.10, improving an already great package for
+interfacing with C.
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.8 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.7 released March, 2017)
+=======================================================================================
+
+See also issues that were resolved_
+
+* New features and cleanups
+
+  * 
+  * 
+
+* Bug Fixes
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+  * 
+
+* RPython improvements
+
+  * 
+  * 
+
+
+Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
+======================================================================
+
+* New features
+
+  * 
+  * 
+
+* Bug Fixes
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+  * PEP 489: Multi-phase extension module initialization
+
+.. _resolved: whatsnew-pypy2-5.8.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,8 @@
+==========================
+What's new in PyPy2.7 5.9+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.8.0
+.. startrev: 0c91b5f4f275
+
+

From pypy.commits at gmail.com  Tue May 30 09:53:34 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: add this to default too
Message-ID: <592d795e.92141c0a.a7d1d.9c81@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91460:3fb4e5e0b6fd
Date: 2017-05-30 16:51 +0300
http://bitbucket.org/pypy/pypy/changeset/3fb4e5e0b6fd/

Log:	add this to default too

diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -0,0 +1,32 @@
+=========================
+What's new in PyPy3 5.7+
+=========================
+
+.. this is the revision after release-pypy3.3-5.7.x was branched
+.. startrev: afbf09453369
+
+.. branch: mtest
+Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+instead of our outdated regrverbose.py script.
+
+.. branch: win32-faulthandler
+
+Enable the 'faulthandler' module on Windows;
+this unblocks the Python test suite.
+
+.. branch: superjumbo
+
+Implement posix.posix_fallocate() and posix.posix_fadvise()
+
+.. branch: py3.5-mac-translate
+
+Fix for different posix primitives on MacOS
+
+.. branch: PyBuffer
+
+Internal refactoring of memoryviews and buffers, fixing some related
+performance issues.
+
+.. branch: jumbojet
+
+Add sched_get min/max to rposix

From pypy.commits at gmail.com  Tue May 30 09:53:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 06:53:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix first commit after release
Message-ID: <592d7960.83821c0a.d29c6.133a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91461:f8822b006de1
Date: 2017-05-30 16:52 +0300
http://bitbucket.org/pypy/pypy/changeset/f8822b006de1/

Log:	fix first commit after release

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,6 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.8.0
-.. startrev: 0c91b5f4f275
+.. startrev: 558bd00b3dd8
 
 

From pypy.commits at gmail.com  Tue May 30 10:36:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 30 May 2017 07:36:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: start to summarize changes since 5.7.0,
 read up to changeset 198dc138680f
Message-ID: <592d8374.4c3a1c0a.7e266.7258@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91462:77bf5cbc29e9
Date: 2017-05-30 17:35 +0300
http://bitbucket.org/pypy/pypy/changeset/77bf5cbc29e9/

Log:	start to summarize changes since 5.7.0, read up to changeset
	198dc138680f

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -83,46 +83,70 @@
 
 * New features and cleanups
 
-  * 
-  * 
+  * Implement PyModule_New, 
+  * Fix for multiple inheritance in app-level for C-API defined classes
+  * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
+  * Document more differences with CPython here_
+  * Add native PyPy support to profile frames in vmprof
+  * Fix an issue with Exception order on failed import
+  * Fix for a corner case of __future__ imports
 
 * Bug Fixes
 
-  * 
-  * 
+  * Correctly handle dict.pop where the popping key is not the same type as the
+    dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
+  * Improve our file's universal newline .readline implementation for
+    ``\n``, ``\r`` confusion
 
 * Performance improvements:
 
-  * 
-  * 
+  * Tweaks made to improve performance by reducing the number of guards
+    inserted in jitted code, based on feedback from users
+  * Add garbage collector memory pressure to some c-level allocations
 
 * RPython improvements
 
-  * 
-  * 
+  * Improve the default shadowstack garbage collector, fixing a crash with
+    multithreaded code and other issues
+  * Make sure lstrip consumes the entire string
+  * Support posix_fallocate and posix_fadvise, expose them on PyPy3.5
+  * Test and fix for int_and() propagating wrong bounds
+  * Improve the generated machine code by tracking the (constant) value of
+    r11 across intructions.  This lets us avoid reloading r11 with another
+    (apparently slowish) "movabs" instruction, replacing it with either
+    nothing or a cheaper variant.
+  * Performance tweaks in the x86 JIT-generated machine code: rarely taken
+    blocks are moved off-line.  Also, the temporary register used to contain
+    large constants is reused across instructions. This helps CPUs branch
+    predictor
 
+.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
 
 Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
 ======================================================================
 
 * New features
 
-  * 
-  * 
+  * Implement main part of PEP 489 (multi-phase extension module initialization)
+  * Add docstrings to various modules and functions
 
 * Bug Fixes
 
-  * 
-  * 
+  * Fix inconsistencies in the xml.etree.ElementTree.Element class, which on
+    CPython is hidden by the C version from '_elementree'.
+  * OSError(None,None) is different from OSError()
+  * Get closer to supporting 32 bit windows, translation now succeeds and most
+    lib-python/3/test runs
 
 * Performance improvements:
 
-  * 
+  * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+    instead of our outdated regrverbose.py script
+  * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
-  * PEP 489: Multi-phase extension module initialization
 
 .. _resolved: whatsnew-pypy2-5.8.0.html
 

From pypy.commits at gmail.com  Tue May 30 13:10:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 10:10:22 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Documentation
Message-ID: <592da77e.51d81c0a.ac8ab.d3d8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2946:927b4c3bbd9a
Date: 2017-05-30 19:09 +0200
http://bitbucket.org/cffi/cffi/changeset/927b4c3bbd9a/

Log:	Documentation

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -228,6 +228,12 @@
         assert type(x[5]) is complex
         assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
         assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
     py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -642,16 +642,15 @@
 
 * Any ``__attribute__`` or ``#pragma pack(n)``
 
-* Additional types: complex numbers, special-size floating and fixed
-  point types, vector types, and so on.  You might be able to access an
-  array of complex numbers by declaring it as an array of ``struct
-  my_complex { double real, imag; }``, but in general you should declare
-  them as ``struct { ...; }`` and cannot access them directly.  This
-  means that you cannot call any function which has an argument or
-  return value of this type (this would need added support in libffi).
-  You need to write wrapper functions in C, e.g. ``void
-  foo_wrapper(struct my_complex c) { foo(c.real + c.imag*1j); }``, and
-  call ``foo_wrapper`` rather than ``foo`` directly.
+* Additional types: special-size floating and fixed
+  point types, vector types, and so on.
+
+* The C99 types ``float _Complex`` and ``double _Complex`` are supported
+  by cffi since version 1.11, but not libffi: you cannot call C
+  functions with complex arguments or return value, except if they are
+  directly API-mode functions.  The type ``long double _Complex`` is not
+  supported at all (declare and use it as if it were an array of two
+  ``long double``, and write wrapper functions in C with set_source()).
 
 Note that declarations like ``int field[];`` in
 structures are interpreted as variable-length structures.  Declarations
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -618,8 +618,8 @@
 |    C type     |   writing into         | reading from     |other operations|
 +===============+========================+==================+================+
 |   integers    | an integer or anything | a Python int or  | int(), bool()  |
-|   and enums   | on which int() works   | long, depending  | `(******)`,    |
-|   `(*****)`   | (but not a float!).    | on the type      | ``<``          |
+|   and enums   | on which int() works   | long, depending  | `[6]`,         |
+|   `[5]`       | (but not a float!).    | on the type      | ``<``          |
 |               | Must be within range.  | (ver. 1.10: or a |                |
 |               |                        | bool)            |                |
 +---------------+------------------------+------------------+----------------+
@@ -636,14 +636,19 @@
 +---------------+------------------------+------------------+----------------+
 |``long double``| another <cdata> with   | a <cdata>, to    | float(), int(),|
 |               | a ``long double``, or  | avoid loosing    | bool()         |
-|               | anything on which      | precision `(***)`|                |
+|               | anything on which      | precision `[3]`  |                |
 |               | float() works          |                  |                |
 +---------------+------------------------+------------------+----------------+
-|  pointers     | another <cdata> with   | a <cdata>        |``[]`` `(****)`,|
+| ``float``     | a complex number       | a Python complex | complex(),     |
+| ``_Complex``, | or anything on which   | number           | bool()         |
+| ``double``    | complex() works        |                  | `[7]`          |
+| ``_Complex``  |                        |                  |                |
++---------------+------------------------+------------------+----------------+
+|  pointers     | another <cdata> with   | a <cdata>        |``[]`` `[4]`,   |
 |               | a compatible type (i.e.|                  |``+``, ``-``,   |
 |               | same type              |                  |bool()          |
 |               | or ``void*``, or as an |                  |                |
-|               | array instead) `(*)`   |                  |                |
+|               | array instead) `[1]`   |                  |                |
 +---------------+------------------------+                  |                |
 |  ``void *``   | another <cdata> with   |                  |                |
 |               | any pointer or array   |                  |                |
@@ -655,10 +660,10 @@
 |               |                        |                  | struct fields  |
 +---------------+------------------------+                  +----------------+
 | function      | same as pointers       |                  | bool(),        |
-| pointers      |                        |                  | call `(**)`    |
+| pointers      |                        |                  | call `[2]`     |
 +---------------+------------------------+------------------+----------------+
 |  arrays       | a list or tuple of     | a <cdata>        |len(), iter(),  |
-|               | items                  |                  |``[]`` `(****)`,|
+|               | items                  |                  |``[]`` `[4]`,   |
 |               |                        |                  |``+``, ``-``    |
 +---------------+------------------------+                  +----------------+
 | ``char[]``,   | same as arrays, or a   |                  | len(), iter(), |
@@ -680,7 +685,7 @@
 |               | with at most one field |                  | fields         |
 +---------------+------------------------+------------------+----------------+
 
-`(*)` ``item *`` is ``item[]`` in function arguments:
+`[1]` ``item *`` is ``item[]`` in function arguments:
 
    In a function declaration, as per the C standard, a ``item *``
    argument is identical to a ``item[]`` argument (and ``ffi.cdef()``
@@ -701,7 +706,7 @@
    (On PyPy, this optimization is only available since PyPy 5.4
    with CFFI 1.8.)
 
-`(**)` C function calls are done with the GIL released.
+`[2]` C function calls are done with the GIL released.
 
    Note that we assume that the called functions are *not* using the
    Python API from Python.h.  For example, we don't check afterwards
@@ -713,7 +718,7 @@
    ``libpypy-c.dll`` on their own.  But really, don't do that in the
    first place.)
 
-`(***)` ``long double`` support:
+`[3]` ``long double`` support:
 
    We keep ``long double`` values inside a cdata object to avoid
    loosing precision.  Normal Python floating-point numbers only
@@ -724,7 +729,7 @@
    and use a family of C functions like ``long double add(long double
    a, long double b);``.
 
-`(****)` Slicing with ``x[start:stop]``:
+`[4]` Slicing with ``x[start:stop]``:
 
    Slicing is allowed, as long as you specify explicitly both ``start``
    and ``stop`` (and don't give any ``step``).  It gives a cdata
@@ -738,7 +743,7 @@
    say ``chararray[10:15] = "hello"``, but the assigned string must be of
    exactly the correct length; no implicit null character is added.)
 
-`(*****)` Enums are handled like ints:
+`[5]` Enums are handled like ints:
 
    Like C, enum types are mostly int types (unsigned or signed, int or
    long; note that GCC's first choice is unsigned).  Reading an enum
@@ -747,7 +752,7 @@
    lib.FOO``.  If you really want to get their value as a string, use
    ``ffi.string(ffi.cast("the_enum_type", x.field))``.
 
-`(******)` bool() on a primitive cdata:
+`[6]` bool() on a primitive cdata:
 
    *New in version 1.7.*  In previous versions, it only worked on
    pointers; for primitives it always returned True.
@@ -760,6 +765,13 @@
    Also, when converting from a byte string to a ``_Bool[]``, only the
    bytes ``\x00`` and ``\x01`` are accepted.
 
+`[7]` libffi does not support complex numbers:
+
+   *New in version 1.11:* CFFI now supports complex numbers directly.
+   Note however that libffi does not.  This means that C functions that
+   take directly as argument types or return type a complex type cannot
+   be called by CFFI, unless they are directly using the API mode.
+
 .. _file:
 
 Support for FILE

From pypy.commits at gmail.com  Tue May 30 13:10:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 10:10:24 -0700 (PDT)
Subject: [pypy-commit] cffi sirtom67/float_complex: Close branch,
 ready to merge
Message-ID: <592da780.8388df0a.3a66.18ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: sirtom67/float_complex
Changeset: r2947:16c9ce0f1b82
Date: 2017-05-30 19:09 +0200
http://bitbucket.org/cffi/cffi/changeset/16c9ce0f1b82/

Log:	Close branch, ready to merge


From pypy.commits at gmail.com  Tue May 30 13:10:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 10:10:26 -0700 (PDT)
Subject: [pypy-commit] cffi default: Merge branch sirtom67/float_complex,
 adding support for complex numbers.
Message-ID: <592da782.11addf0a.2b75b.c990@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2948:71fd95269a8f
Date: 2017-05-30 19:10 +0200
http://bitbucket.org/cffi/cffi/changeset/71fd95269a8f/

Log:	Merge branch sirtom67/float_complex, adding support for complex
	numbers. Thanks sirtom67 for doing most of the work!

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -116,36 +116,38 @@
 /************************************************************/
 
 /* base type flag: exactly one of the following: */
-#define CT_PRIMITIVE_SIGNED   1    /* signed integer */
-#define CT_PRIMITIVE_UNSIGNED 2    /* unsigned integer */
-#define CT_PRIMITIVE_CHAR     4    /* char, wchar_t */
-#define CT_PRIMITIVE_FLOAT    8    /* float, double, long double */
-#define CT_POINTER           16    /* pointer, excluding ptr-to-func */
-#define CT_ARRAY             32    /* array */
-#define CT_STRUCT            64    /* struct */
-#define CT_UNION            128    /* union */
-#define CT_FUNCTIONPTR      256    /* pointer to function */
-#define CT_VOID             512    /* void */
+#define CT_PRIMITIVE_SIGNED   0x001   /* signed integer */
+#define CT_PRIMITIVE_UNSIGNED 0x002   /* unsigned integer */
+#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t */
+#define CT_PRIMITIVE_FLOAT    0x008   /* float, double, long double */
+#define CT_POINTER            0x010   /* pointer, excluding ptr-to-func */
+#define CT_ARRAY              0x020   /* array */
+#define CT_STRUCT             0x040   /* struct */
+#define CT_UNION              0x080   /* union */
+#define CT_FUNCTIONPTR        0x100   /* pointer to function */
+#define CT_VOID               0x200   /* void */
+#define CT_PRIMITIVE_COMPLEX  0x400   /* float _Complex, double _Complex */
 
 /* other flags that may also be set in addition to the base flag: */
-#define CT_IS_VOIDCHAR_PTR       1024
-#define CT_PRIMITIVE_FITS_LONG   2048
-#define CT_IS_OPAQUE             4096
-#define CT_IS_ENUM               8192
-#define CT_IS_PTR_TO_OWNED      16384   /* only owned if CDataOwning_Type */
-#define CT_CUSTOM_FIELD_POS     32768
-#define CT_IS_LONGDOUBLE        65536
-#define CT_IS_BOOL             131072
-#define CT_IS_FILE             262144
-#define CT_IS_VOID_PTR         524288
-#define CT_WITH_VAR_ARRAY     1048576
-#define CT_IS_UNSIZED_CHAR_A  2097152
-#define CT_LAZY_FIELD_LIST    4194304
-#define CT_WITH_PACKED_CHANGE 8388608
+#define CT_IS_VOIDCHAR_PTR     0x00001000
+#define CT_PRIMITIVE_FITS_LONG 0x00002000
+#define CT_IS_OPAQUE           0x00004000
+#define CT_IS_ENUM             0x00008000
+#define CT_IS_PTR_TO_OWNED     0x00010000 /* only owned if CDataOwning_Type */
+#define CT_CUSTOM_FIELD_POS    0x00020000
+#define CT_IS_LONGDOUBLE       0x00040000
+#define CT_IS_BOOL             0x00080000
+#define CT_IS_FILE             0x00100000
+#define CT_IS_VOID_PTR         0x00200000
+#define CT_WITH_VAR_ARRAY      0x00400000
+#define CT_IS_UNSIZED_CHAR_A   0x00800000
+#define CT_LAZY_FIELD_LIST     0x01000000
+#define CT_WITH_PACKED_CHANGE  0x02000000
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
-                           CT_PRIMITIVE_FLOAT)
+                           CT_PRIMITIVE_FLOAT |         \
+                           CT_PRIMITIVE_COMPLEX)
 
 typedef struct _ctypedescr {
     PyObject_VAR_HEAD
@@ -883,6 +885,26 @@
     return 0;
 }
 
+static Py_complex
+read_raw_complex_data(char *target, int size)
+{
+    Py_complex r = {0.0, 0.0};
+    if (size == 2*sizeof(float)) {
+        float real_part, imag_part;
+        memcpy(&real_part, target + 0,             sizeof(float));
+        memcpy(&imag_part, target + sizeof(float), sizeof(float));
+        r.real = real_part;
+        r.imag = imag_part;
+        return r;
+    }
+    if (size == 2*sizeof(double)) {
+        memcpy(&r, target, 2*sizeof(double));
+        return r;
+    }
+    Py_FatalError("read_raw_complex_data: bad complex size");
+    return r;
+}
+
 static void
 write_raw_float_data(char *target, double source, int size)
 {
@@ -898,6 +920,25 @@
     _write_raw_data(long double);
 }
 
+#define _write_raw_complex_data(type)                      \
+    do {                                                   \
+        if (size == 2*sizeof(type)) {                      \
+            type r = (type)source.real;                    \
+            type i = (type)source.imag;                    \
+            memcpy(target, &r, sizeof(type));              \
+            memcpy(target+sizeof(type), &i, sizeof(type)); \
+            return;                                        \
+        }                                                  \
+    } while(0)
+
+static void
+write_raw_complex_data(char *target, Py_complex source, int size)
+{
+    _write_raw_complex_data(float);
+    _write_raw_complex_data(double);
+    Py_FatalError("write_raw_complex_data: bad complex size");
+}
+
 static PyObject *
 new_simple_cdata(char *data, CTypeDescrObject *ct)
 {
@@ -1015,6 +1056,10 @@
             return _my_PyUnicode_FromWideChar((wchar_t *)data, 1);
 #endif
     }
+    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = read_raw_complex_data(data, ct->ct_size);
+        return PyComplex_FromCComplex(value);
+    }
 
     PyErr_Format(PyExc_SystemError,
                  "convert_to_object: '%s'", ct->ct_name);
@@ -1519,6 +1564,13 @@
         }
         return convert_struct_from_object(data, ct, init, NULL);
     }
+    if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = PyComplex_AsCComplex(init);
+        if (PyErr_Occurred())
+            return -1;
+        write_raw_complex_data(data, value, ct->ct_size);
+        return 0;
+    }
     PyErr_Format(PyExc_SystemError,
                  "convert_from_object: '%s'", ct->ct_name);
     return -1;
@@ -1956,6 +2008,11 @@
                 return read_raw_longdouble_data(cd->c_data) != 0.0;
             return read_raw_float_data(cd->c_data, cd->c_type->ct_size) != 0.0;
         }
+        if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
+            Py_complex value = read_raw_complex_data(cd->c_data,
+                                                     cd->c_type->ct_size);
+            return value.real != 0.0 || value.imag != 0.0;
+        }
     }
     return cd->c_data != NULL;
 }
@@ -2904,6 +2961,24 @@
     }
 }
 
+static PyObject *cdata_complex(PyObject *cd_, PyObject *noarg)
+{
+    CDataObject *cd = (CDataObject *)cd_;
+
+    if (cd->c_type->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        Py_complex value = read_raw_complex_data(cd->c_data, cd->c_type->ct_size);
+        PyObject *op = PyComplex_FromCComplex(value);
+        return op;
+    }
+    /* <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+       calling complex(), just like <cdata 'int'> cannot be directly
+       converted by calling float() */
+
+    PyErr_Format(PyExc_TypeError, "complex() not supported on cdata '%s'",
+                 cd->c_type->ct_name);
+    return NULL;
+}
+
 static PyObject *cdata_iter(CDataObject *);
 
 static PyNumberMethods CData_as_number = {
@@ -2953,8 +3028,9 @@
 };
 
 static PyMethodDef cdata_methods[] = {
-    {"__dir__",   cdata_dir,      METH_NOARGS},
-    {NULL,        NULL}           /* sentinel */
+    {"__dir__",     cdata_dir,      METH_NOARGS},
+    {"__complex__", cdata_complex,  METH_NOARGS},
+    {NULL,          NULL}           /* sentinel */
 };
 
 static PyTypeObject CData_Type = {
@@ -3587,6 +3663,31 @@
     return cd;
 }
 
+/* returns -1 if cannot cast, 0 if we don't get a value, 1 if we do */
+static int check_bytes_for_float_compatible(PyObject *io, double *out_value)
+{
+    if (PyBytes_Check(io)) {
+        if (PyBytes_GET_SIZE(io) != 1) {
+            Py_DECREF(io);
+            return -1;
+        }
+        *out_value = (unsigned char)PyBytes_AS_STRING(io)[0];
+        return 1;
+    }
+#if HAVE_WCHAR_H
+    else if (PyUnicode_Check(io)) {
+        wchar_t ordinal;
+        if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
+            Py_DECREF(io);
+            return -1;
+        }
+        *out_value = (long)ordinal;
+        return 1;
+    }
+#endif
+    return 0;
+}
+
 static PyObject *do_cast(CTypeDescrObject *ct, PyObject *ob)
 {
     CDataObject *cd;
@@ -3628,6 +3729,7 @@
         /* cast to a float */
         double value;
         PyObject *io;
+        int res;
 
         if (CData_Check(ob)) {
             CDataObject *cdsrc = (CDataObject *)ob;
@@ -3643,37 +3745,23 @@
             Py_INCREF(io);
         }
 
-        if (PyBytes_Check(io)) {
-            if (PyBytes_GET_SIZE(io) != 1) {
-                Py_DECREF(io);
-                goto cannot_cast;
-            }
-            value = (unsigned char)PyBytes_AS_STRING(io)[0];
-        }
-#if HAVE_WCHAR_H
-        else if (PyUnicode_Check(io)) {
-            wchar_t ordinal;
-            if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
-                Py_DECREF(io);
-                goto cannot_cast;
-            }
-            value = (long)ordinal;
-        }
-#endif
-        else if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
+        res = check_bytes_for_float_compatible(io, &value);
+        if (res == -1)
+            goto cannot_cast;
+        if (res == 0) {
+            if ((ct->ct_flags & CT_IS_LONGDOUBLE) &&
                  CData_Check(io) &&
                  (((CDataObject *)io)->c_type->ct_flags & CT_IS_LONGDOUBLE)) {
-            long double lvalue;
-            char *data = ((CDataObject *)io)->c_data;
-            /*READ(data, sizeof(long double)*/
-            lvalue = read_raw_longdouble_data(data);
-            Py_DECREF(io);
-            cd = _new_casted_primitive(ct);
-            if (cd != NULL)
-                write_raw_longdouble_data(cd->c_data, lvalue);
-            return (PyObject *)cd;
-        }
-        else {
+                long double lvalue;
+                char *data = ((CDataObject *)io)->c_data;
+                /*READ(data, sizeof(long double)*/
+                lvalue = read_raw_longdouble_data(data);
+                Py_DECREF(io);
+                cd = _new_casted_primitive(ct);
+                if (cd != NULL)
+                    write_raw_longdouble_data(cd->c_data, lvalue);
+                return (PyObject *)cd;
+            }
             value = PyFloat_AsDouble(io);
         }
         Py_DECREF(io);
@@ -3689,6 +3777,45 @@
         }
         return (PyObject *)cd;
     }
+    else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
+        /* cast to a complex */
+        Py_complex value;
+        PyObject *io;
+        int res;
+
+        if (CData_Check(ob)) {
+            CDataObject *cdsrc = (CDataObject *)ob;
+
+            if (!(cdsrc->c_type->ct_flags & CT_PRIMITIVE_ANY))
+                goto cannot_cast;
+            io = convert_to_object(cdsrc->c_data, cdsrc->c_type);
+            if (io == NULL)
+                return NULL;
+        }
+        else {
+            io = ob;
+            Py_INCREF(io);
+        }
+
+        res = check_bytes_for_float_compatible(io, &value.real);
+        if (res == -1)
+            goto cannot_cast;
+        if (res == 1) {
+            // got it from string
+            value.imag = 0.0;
+        } else {
+            value = PyComplex_AsCComplex(io);
+        }
+        Py_DECREF(io);
+        if (PyErr_Occurred()) {
+            return NULL;
+        }
+        cd = _new_casted_primitive(ct);
+        if (cd != NULL) {
+            write_raw_complex_data(cd->c_data, value, ct->ct_size);
+        }
+        return (PyObject *)cd;
+    }
     else {
         PyErr_Format(PyExc_TypeError, "cannot cast to ctype '%s'",
                      ct->ct_name);
@@ -3954,6 +4081,11 @@
     return NULL;
 }
 
+/* according to the C standard, these types should be equivalent to the
+   _Complex types for the purposes of storage (not arguments in calls!) */
+typedef float cffi_float_complex_t[2];
+typedef double cffi_double_complex_t[2];
+
 static PyObject *new_primitive_type(const char *name)
 {
 #define ENUM_PRIMITIVE_TYPES                                    \
@@ -3971,6 +4103,8 @@
        EPTYPE(f, float, CT_PRIMITIVE_FLOAT )                    \
        EPTYPE(d, double, CT_PRIMITIVE_FLOAT )                   \
        EPTYPE(ld, long double, CT_PRIMITIVE_FLOAT | CT_IS_LONGDOUBLE ) \
+       EPTYPE2(fc, "float _Complex", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX ) \
+       EPTYPE2(dc, "double _Complex", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX ) \
        ENUM_PRIMITIVE_TYPES_WCHAR                               \
        EPTYPE(b, _Bool, CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL )    \
      /* the following types are not primitive in the C sense */ \
@@ -4081,6 +4215,13 @@
         else
             goto bad_ffi_type;
     }
+    else if (ptypes->flags & CT_PRIMITIVE_COMPLEX) {
+        /* As of March 2017, still no libffi support for complex.
+           It fails silently if we try to use ffi_type_complex_float
+           or ffi_type_complex_double.  Better not use it at all.
+         */
+        ffitype = NULL;
+    }
     else {
         switch (ptypes->size) {
         case 1: ffitype = &ffi_type_uint8; break;
@@ -4773,7 +4914,7 @@
 {
     const char *place = is_result_type ? "return value" : "argument";
 
-    if (ct->ct_flags & CT_PRIMITIVE_ANY) {
+    if (ct->ct_flags & (CT_PRIMITIVE_ANY & ~CT_PRIMITIVE_COMPLEX)) {
         return (ffi_type *)ct->ct_extra;
     }
     else if (ct->ct_flags & (CT_POINTER|CT_FUNCTIONPTR)) {
@@ -4899,9 +5040,16 @@
         return NULL;
     }
     else {
+        char *extra = "";
+        if (ct->ct_flags & CT_PRIMITIVE_COMPLEX)
+            extra = " (the support for complex types inside libffi "
+                    "is mostly missing at this point, so CFFI only "
+                    "supports complex types as arguments or return "
+                    "value in API-mode functions)";
+
         PyErr_Format(PyExc_NotImplementedError,
-                     "ctype '%s' (size %zd) not supported as %s",
-                     ct->ct_name, ct->ct_size, place);
+                     "ctype '%s' (size %zd) not supported as %s%s",
+                     ct->ct_name, ct->ct_size, place, extra);
         return NULL;
     }
 }
@@ -6579,6 +6727,20 @@
     return -42;
 }
 
+#if 0   /* libffi doesn't properly support complexes currently */
+        /* also, MSVC might not support _Complex... */
+        /* if this is enabled one day, remember to also add _Complex
+         * arguments in addition to return values. */
+static float _Complex _testfunc24(float a, float b)
+{
+    return a + I*2.0*b;
+}
+static double _Complex _testfunc25(double a, double b)
+{
+    return a + I*2.0*b;
+}
+#endif
+
 static PyObject *b__testfunc(PyObject *self, PyObject *args)
 {
     /* for testing only */
@@ -6611,6 +6773,10 @@
     case 21: f = &_testfunc21; break;
     case 22: f = &_testfunc22; break;
     case 23: f = &_testfunc23; break;
+#if 0
+    case 24: f = &_testfunc24; break;
+    case 25: f = &_testfunc25; break;
+#endif
     default:
         PyErr_SetNone(PyExc_ValueError);
         return NULL;
diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -25,7 +25,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -159,6 +159,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -601,6 +602,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -656,6 +658,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -666,6 +670,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -719,9 +724,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -788,6 +795,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok, "_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -151,6 +151,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
     };
     PyObject *x;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -185,37 +185,56 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
-        assert bool(cast(p, 0))
+        p = new_primitive_type(name + " _Complex")
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
-        assert cast(p, -1.1j) != cast(p, -1.1j)
+        assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1116,6 +1135,34 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
+def test_call_function_25():
+    BDouble = new_primitive_type("double")
+    BDoubleComplex = new_primitive_type("double _Complex")
+    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -642,16 +642,15 @@
 
 * Any ``__attribute__`` or ``#pragma pack(n)``
 
-* Additional types: complex numbers, special-size floating and fixed
-  point types, vector types, and so on.  You might be able to access an
-  array of complex numbers by declaring it as an array of ``struct
-  my_complex { double real, imag; }``, but in general you should declare
-  them as ``struct { ...; }`` and cannot access them directly.  This
-  means that you cannot call any function which has an argument or
-  return value of this type (this would need added support in libffi).
-  You need to write wrapper functions in C, e.g. ``void
-  foo_wrapper(struct my_complex c) { foo(c.real + c.imag*1j); }``, and
-  call ``foo_wrapper`` rather than ``foo`` directly.
+* Additional types: special-size floating and fixed
+  point types, vector types, and so on.
+
+* The C99 types ``float _Complex`` and ``double _Complex`` are supported
+  by cffi since version 1.11, but not libffi: you cannot call C
+  functions with complex arguments or return value, except if they are
+  directly API-mode functions.  The type ``long double _Complex`` is not
+  supported at all (declare and use it as if it were an array of two
+  ``long double``, and write wrapper functions in C with set_source()).
 
 Note that declarations like ``int field[];`` in
 structures are interpreted as variable-length structures.  Declarations
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -618,8 +618,8 @@
 |    C type     |   writing into         | reading from     |other operations|
 +===============+========================+==================+================+
 |   integers    | an integer or anything | a Python int or  | int(), bool()  |
-|   and enums   | on which int() works   | long, depending  | `(******)`,    |
-|   `(*****)`   | (but not a float!).    | on the type      | ``<``          |
+|   and enums   | on which int() works   | long, depending  | `[6]`,         |
+|   `[5]`       | (but not a float!).    | on the type      | ``<``          |
 |               | Must be within range.  | (ver. 1.10: or a |                |
 |               |                        | bool)            |                |
 +---------------+------------------------+------------------+----------------+
@@ -636,14 +636,19 @@
 +---------------+------------------------+------------------+----------------+
 |``long double``| another <cdata> with   | a <cdata>, to    | float(), int(),|
 |               | a ``long double``, or  | avoid loosing    | bool()         |
-|               | anything on which      | precision `(***)`|                |
+|               | anything on which      | precision `[3]`  |                |
 |               | float() works          |                  |                |
 +---------------+------------------------+------------------+----------------+
-|  pointers     | another <cdata> with   | a <cdata>        |``[]`` `(****)`,|
+| ``float``     | a complex number       | a Python complex | complex(),     |
+| ``_Complex``, | or anything on which   | number           | bool()         |
+| ``double``    | complex() works        |                  | `[7]`          |
+| ``_Complex``  |                        |                  |                |
++---------------+------------------------+------------------+----------------+
+|  pointers     | another <cdata> with   | a <cdata>        |``[]`` `[4]`,   |
 |               | a compatible type (i.e.|                  |``+``, ``-``,   |
 |               | same type              |                  |bool()          |
 |               | or ``void*``, or as an |                  |                |
-|               | array instead) `(*)`   |                  |                |
+|               | array instead) `[1]`   |                  |                |
 +---------------+------------------------+                  |                |
 |  ``void *``   | another <cdata> with   |                  |                |
 |               | any pointer or array   |                  |                |
@@ -655,10 +660,10 @@
 |               |                        |                  | struct fields  |
 +---------------+------------------------+                  +----------------+
 | function      | same as pointers       |                  | bool(),        |
-| pointers      |                        |                  | call `(**)`    |
+| pointers      |                        |                  | call `[2]`     |
 +---------------+------------------------+------------------+----------------+
 |  arrays       | a list or tuple of     | a <cdata>        |len(), iter(),  |
-|               | items                  |                  |``[]`` `(****)`,|
+|               | items                  |                  |``[]`` `[4]`,   |
 |               |                        |                  |``+``, ``-``    |
 +---------------+------------------------+                  +----------------+
 | ``char[]``,   | same as arrays, or a   |                  | len(), iter(), |
@@ -680,7 +685,7 @@
 |               | with at most one field |                  | fields         |
 +---------------+------------------------+------------------+----------------+
 
-`(*)` ``item *`` is ``item[]`` in function arguments:
+`[1]` ``item *`` is ``item[]`` in function arguments:
 
    In a function declaration, as per the C standard, a ``item *``
    argument is identical to a ``item[]`` argument (and ``ffi.cdef()``
@@ -701,7 +706,7 @@
    (On PyPy, this optimization is only available since PyPy 5.4
    with CFFI 1.8.)
 
-`(**)` C function calls are done with the GIL released.
+`[2]` C function calls are done with the GIL released.
 
    Note that we assume that the called functions are *not* using the
    Python API from Python.h.  For example, we don't check afterwards
@@ -713,7 +718,7 @@
    ``libpypy-c.dll`` on their own.  But really, don't do that in the
    first place.)
 
-`(***)` ``long double`` support:
+`[3]` ``long double`` support:
 
    We keep ``long double`` values inside a cdata object to avoid
    loosing precision.  Normal Python floating-point numbers only
@@ -724,7 +729,7 @@
    and use a family of C functions like ``long double add(long double
    a, long double b);``.
 
-`(****)` Slicing with ``x[start:stop]``:
+`[4]` Slicing with ``x[start:stop]``:
 
    Slicing is allowed, as long as you specify explicitly both ``start``
    and ``stop`` (and don't give any ``step``).  It gives a cdata
@@ -738,7 +743,7 @@
    say ``chararray[10:15] = "hello"``, but the assigned string must be of
    exactly the correct length; no implicit null character is added.)
 
-`(*****)` Enums are handled like ints:
+`[5]` Enums are handled like ints:
 
    Like C, enum types are mostly int types (unsigned or signed, int or
    long; note that GCC's first choice is unsigned).  Reading an enum
@@ -747,7 +752,7 @@
    lib.FOO``.  If you really want to get their value as a string, use
    ``ffi.string(ffi.cast("the_enum_type", x.field))``.
 
-`(******)` bool() on a primitive cdata:
+`[6]` bool() on a primitive cdata:
 
    *New in version 1.7.*  In previous versions, it only worked on
    pointers; for primitives it always returned True.
@@ -760,6 +765,13 @@
    Also, when converting from a byte string to a ``_Bool[]``, only the
    bytes ``\x00`` and ``\x01`` are accepted.
 
+`[7]` libffi does not support complex numbers:
+
+   *New in version 1.11:* CFFI now supports complex numbers directly.
+   Note however that libffi does not.  This means that C functions that
+   take directly as argument types or return type a complex type cannot
+   be called by CFFI, unless they are directly using the API mode.
+
 .. _file:
 
 Support for FILE
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -239,15 +239,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1704,6 +1704,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'double _Complex',
+            'float _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/testing/cffi1/test_parse_c_type.py b/testing/cffi1/test_parse_c_type.py
--- a/testing/cffi1/test_parse_c_type.py
+++ b/testing/cffi1/test_parse_c_type.py
@@ -155,6 +155,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -280,6 +282,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/testing/cffi1/test_realize_c_type.py b/testing/cffi1/test_realize_c_type.py
--- a/testing/cffi1/test_realize_c_type.py
+++ b/testing/cffi1/test_realize_c_type.py
@@ -47,7 +47,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2001,6 +2001,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -219,15 +219,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Tue May 30 13:19:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 10:19:19 -0700 (PDT)
Subject: [pypy-commit] cffi default: pypy tweak
Message-ID: <592da997.cf921c0a.c01c0.8412@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2949:0ff0669fce9d
Date: 2017-05-30 19:19 +0200
http://bitbucket.org/cffi/cffi/changeset/0ff0669fce9d/

Log:	pypy tweak

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -736,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)

From pypy.commits at gmail.com  Tue May 30 13:22:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 30 May 2017 10:22:45 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: A branch to implement complex
 numbers from cffi 1.11
Message-ID: <592daa65.84871c0a.e1b81.4e69@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91463:4a7c80c655ff
Date: 2017-05-30 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/4a7c80c655ff/

Log:	A branch to implement complex numbers from cffi 1.11


From pypy.commits at gmail.com  Wed May 31 06:27:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 03:27:43 -0700 (PDT)
Subject: [pypy-commit] cffi default: Whatsnew
Message-ID: <592e9a9f.c48d1c0a.f015a.ec99@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2950:f6f8d7ce7ff2
Date: 2017-05-31 12:26 +0200
http://bitbucket.org/cffi/cffi/changeset/f6f8d7ce7ff2/

Log:	Whatsnew

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,15 @@
 ======================
 
 
+v1.11
+=====
+
+* Support the C99 types ``float _Complex`` and ``double _Complex``.
+  Note that libffi doesn't support them, which means that in the ABI
+  mode you still cannot call C functions that take complex numbers
+  directly as arguments or return type.
+
+
 v1.10.1
 =======
 

From pypy.commits at gmail.com  Wed May 31 06:27:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 03:27:45 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: A branch for issue #315
Message-ID: <592e9aa1.b885df0a.cad2b.6b1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2951:bb0d933723be
Date: 2017-05-31 12:27 +0200
http://bitbucket.org/cffi/cffi/changeset/bb0d933723be/

Log:	A branch for issue #315

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -118,7 +118,7 @@
 /* base type flag: exactly one of the following: */
 #define CT_PRIMITIVE_SIGNED   0x001   /* signed integer */
 #define CT_PRIMITIVE_UNSIGNED 0x002   /* unsigned integer */
-#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t */
+#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t, charN_t */
 #define CT_PRIMITIVE_FLOAT    0x008   /* float, double, long double */
 #define CT_POINTER            0x010   /* pointer, excluding ptr-to-func */
 #define CT_ARRAY              0x020   /* array */
@@ -285,9 +285,7 @@
 # include "file_emulator.h"
 #endif
 
-#ifdef HAVE_WCHAR_H
-# include "wchar_helper.h"
-#endif
+#include "wchar_helper.h"
 
 typedef struct _cffi_allocator_s {
     PyObject *ca_alloc, *ca_free;
@@ -1049,12 +1047,14 @@
     }
     else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(data, ct->ct_size)*/
-        if (ct->ct_size == sizeof(char))
+        switch (ct->ct_size) {
+        case sizeof(char):
             return PyBytes_FromStringAndSize(data, 1);
-#ifdef HAVE_WCHAR_H
-        else
-            return _my_PyUnicode_FromWideChar((wchar_t *)data, 1);
-#endif
+        case 2:
+            return _my_PyUnicode_FromChar16((cffi_char16_t *)data, 1);
+        case 4:
+            return _my_PyUnicode_FromChar32((cffi_char32_t *)data, 1);
+        }
     }
     else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
         Py_complex value = read_raw_complex_data(data, ct->ct_size);
@@ -1133,27 +1133,53 @@
     return -1;
 }
 
-#ifdef HAVE_WCHAR_H
-static wchar_t _convert_to_wchar_t(PyObject *init)
-{
+static cffi_char16_t _convert_to_char16_t(PyObject *init)
+{
+    char err_got[80];
+    err_got[0] = 0;
+
     if (PyUnicode_Check(init)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(init, &ordinal) == 0)
+        cffi_char16_t ordinal;
+        if (_my_PyUnicode_AsSingleChar16(init, &ordinal, err_got) == 0)
             return ordinal;
     }
     if (CData_Check(init) &&
            (((CDataObject *)init)->c_type->ct_flags & CT_PRIMITIVE_CHAR) &&
-           (((CDataObject *)init)->c_type->ct_size == sizeof(wchar_t))) {
+           (((CDataObject *)init)->c_type->ct_size == 2)) {
         char *data = ((CDataObject *)init)->c_data;
-        /*READ(data, sizeof(wchar_t))*/
-        return *(wchar_t *)data;
+        /*READ(data, 2)*/
+        return *(cffi_char16_t *)data;
     }
     PyErr_Format(PyExc_TypeError,
-                 "initializer for ctype 'wchar_t' must be a unicode string "
-                 "of length 1, not %.200s", Py_TYPE(init)->tp_name);
-    return (wchar_t)-1;
-}
-#endif
+                 "initializer for ctype 'char16_t' must be a unicode string "
+                 "of length 1, not %.200s",
+                 err_got[0] == 0 ? Py_TYPE(init)->tp_name : err_got);
+    return (cffi_char16_t)-1;
+}
+
+static cffi_char32_t _convert_to_char32_t(PyObject *init)
+{
+    char err_got[80];
+    err_got[0] = 0;
+
+    if (PyUnicode_Check(init)) {
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(init, &ordinal, err_got) == 0)
+            return ordinal;
+    }
+    if (CData_Check(init) &&
+           (((CDataObject *)init)->c_type->ct_flags & CT_PRIMITIVE_CHAR) &&
+           (((CDataObject *)init)->c_type->ct_size == 4)) {
+        char *data = ((CDataObject *)init)->c_data;
+        /*READ(data, 4)*/
+        return *(cffi_char32_t *)data;
+    }
+    PyErr_Format(PyExc_TypeError,
+                 "initializer for ctype 'char32_t' must be a unicode string "
+                 "of length 1, not %.200s",
+                 err_got[0] == 0 ? Py_TYPE(init)->tp_name : err_got);
+    return (cffi_char32_t)-1;
+}
 
 static int _convert_error(PyObject *init, const char *ct_name,
                           const char *expected)
@@ -1191,7 +1217,7 @@
 convert_from_object_bitfield(char *data, CFieldObject *cf, PyObject *init);
 
 static Py_ssize_t
-get_new_array_length(PyObject **pvalue)
+get_new_array_length(CTypeDescrObject *ctitem, PyObject **pvalue)
 {
     PyObject *value = *pvalue;
 
@@ -1204,7 +1230,12 @@
     }
     else if (PyUnicode_Check(value)) {
         /* from a unicode, we add the null terminator */
-        return _my_PyUnicode_SizeAsWideChar(value) + 1;
+        int length;
+        if (ctitem->ct_size == 4)
+            length = _my_PyUnicode_SizeAsChar32(value);
+        else
+            length = PyUnicode_GET_SIZE(value);
+        return length + 1;
     }
     else {
         Py_ssize_t explicitlength;
@@ -1336,14 +1367,18 @@
             memcpy(data, srcdata, n);
             return 0;
         }
-#ifdef HAVE_WCHAR_H
         else {
             Py_ssize_t n;
             if (!PyUnicode_Check(init)) {
                 expected = "unicode or list or tuple";
                 goto cannot_convert;
             }
-            n = _my_PyUnicode_SizeAsWideChar(init);
+
+            if (ctitem->ct_size == 4)
+                n = _my_PyUnicode_SizeAsChar32(init);
+            else
+                n = _my_PyUnicode_SizeAsChar16(init);
+
             if (ct->ct_length >= 0 && n > ct->ct_length) {
                 PyErr_Format(PyExc_IndexError,
                              "initializer unicode is too long for '%s' "
@@ -1352,10 +1387,12 @@
             }
             if (n != ct->ct_length)
                 n++;
-            _my_PyUnicode_AsWideChar(init, (wchar_t *)data, n);
+            if (ctitem->ct_size == 4)
+                _my_PyUnicode_AsChar32(init, (cffi_char32_t *)data, n);
+            else
+                _my_PyUnicode_AsChar16(init, (cffi_char16_t *)data, n);
             return 0;
         }
-#endif
     }
     else {
         expected = "list or tuple";
@@ -1537,22 +1574,28 @@
         return 0;
     }
     if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
-        if (ct->ct_size == sizeof(char)) {
+        switch (ct->ct_size) {
+        case sizeof(char): {
             int res = _convert_to_char(init);
             if (res < 0)
                 return -1;
             data[0] = res;
             return 0;
         }
-#ifdef HAVE_WCHAR_H
-        else {
-            wchar_t res = _convert_to_wchar_t(init);
-            if (res == (wchar_t)-1 && PyErr_Occurred())
+        case 2: {
+            cffi_char16_t res = _convert_to_char16_t(init);
+            if (res == (cffi_char16_t)-1 && PyErr_Occurred())
                 return -1;
-            *(wchar_t *)data = res;
+            *(cffi_char16_t *)data = res;
             return 0;
         }
-#endif
+        case 4: {
+            int res = _convert_to_char32_t(init);
+            if (res == -1 && PyErr_Occurred())
+                return -1;
+            *(cffi_char32_t *)data = res;
+            return 0;
+        }
     }
     if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
 
@@ -2033,12 +2076,16 @@
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(cd->c_data, cd->c_type->ct_size)*/
-        if (cd->c_type->ct_size == sizeof(char))
+        switch (cd->c_type->ct_size) {
+        case sizeof(char):
             return PyInt_FromLong((unsigned char)cd->c_data[0]);
-#ifdef HAVE_WCHAR_H
-        else
-            return PyInt_FromLong((long)*(wchar_t *)cd->c_data);
-#endif
+        case 2:
+            return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data);
+        case 4:
+            /* NB. cast via int32_t instead of cffi_char32_t, so that
+               we expose a signed result to the user */
+            return PyInt_FromLong((long)*(int32_t *)cd->c_data);
+        }
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
         PyObject *o = cdata_float(cd);
@@ -3626,18 +3673,17 @@
         value = (unsigned char)PyString_AS_STRING(ob)[0];
     }
 #endif
-#ifdef HAVE_WCHAR_H
     else if (PyUnicode_Check(ob)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(ob, &ordinal) < 0) {
+        char err_buf[80];
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(ob, &ordinal, err_buf) < 0) {
             PyErr_Format(PyExc_TypeError,
-                      "cannot cast unicode string of length %zd to ctype '%s'",
-                         PyUnicode_GET_SIZE(ob), ct->ct_name);
+                         "cannot cast %s to ctype '%s'", err_buf, ct->ct_name);
             return NULL;
         }
-        value = (long)ordinal;
-    }
-#endif
+        /* the user sees char32_t being signed, but not char16_t */
+        value = (int32_t)ordinal;
+    }
     else if (PyBytes_Check(ob)) {
         int res = _convert_to_char(ob);
         if (res < 0)
@@ -3674,17 +3720,16 @@
         *out_value = (unsigned char)PyBytes_AS_STRING(io)[0];
         return 1;
     }
-#if HAVE_WCHAR_H
     else if (PyUnicode_Check(io)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
+        char ignored[80];
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) {
             Py_DECREF(io);
             return -1;
         }
-        *out_value = (long)ordinal;
+        *out_value = (int32_t)ordinal;
         return 1;
     }
-#endif
     return 0;
 }
 
@@ -4106,6 +4151,8 @@
        EPTYPE2(fc, "float _Complex", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX ) \
        EPTYPE2(dc, "double _Complex", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX ) \
        ENUM_PRIMITIVE_TYPES_WCHAR                               \
+       EPTYPE2(c16, "char16_t", cffi_char16_t, CT_PRIMITIVE_CHAR ) \
+       EPTYPE2(c32, "char32_t", cffi_char32_t, CT_PRIMITIVE_CHAR ) \
        EPTYPE(b, _Bool, CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL )    \
      /* the following types are not primitive in the C sense */ \
        EPTYPE(i8, int8_t, CT_PRIMITIVE_SIGNED)                  \
@@ -6036,27 +6083,45 @@
             }
             return PyBytes_FromStringAndSize(start, length);
         }
-#ifdef HAVE_WCHAR_H
         else if (cd->c_type->ct_itemdescr->ct_flags & CT_PRIMITIVE_CHAR) {
-            const wchar_t *start = (wchar_t *)cd->c_data;
-            assert(cd->c_type->ct_itemdescr->ct_size == sizeof(wchar_t));
-            if (length < 0) {
-                /*READ(start, sizeof(wchar_t))*/
-                length = 0;
-                while (start[length])
-                    length++;
-                /*READ(start, sizeof(wchar_t) * length)*/
+            switch (cd->c_type->ct_itemdescr->ct_size) {
+            case 2: {
+                const cffi_char16_t *start = (cffi_char16_t *)cd->c_data;
+                if (length < 0) {
+                    /*READ(start, 2)*/
+                    length = 0;
+                    while (start[length])
+                        length++;
+                    /*READ(start, 2 * length)*/
+                }
+                else {
+                    /*READ(start, 2 * length)*/
+                    maxlen = length;
+                    length = 0;
+                    while (length < maxlen && start[length])
+                        length++;
+                }
+                return _my_PyUnicode_FromChar16(start, length);
             }
-            else {
-                /*READ(start, sizeof(wchar_t) * length)*/
-                maxlen = length;
-                length = 0;
-                while (length < maxlen && start[length])
-                    length++;
+            case 4: {
+                const cffi_char32_t *start = (cffi_char32_t *)cd->c_data;
+                if (length < 0) {
+                    /*READ(start, 4)*/
+                    length = 0;
+                    while (start[length])
+                        length++;
+                    /*READ(start, 4 * length)*/
+                }
+                else {
+                    /*READ(start, 4 * length)*/
+                    maxlen = length;
+                    length = 0;
+                    while (length < maxlen && start[length])
+                        length++;
+                }
+                return _my_PyUnicode_FromChar32(start, length);
             }
-            return _my_PyUnicode_FromWideChar(start, length);
-        }
-#endif
+        }
     }
     else if (cd->c_type->ct_flags & CT_IS_ENUM) {
         return convert_cdata_to_enum_string(cd, 0);
@@ -6070,12 +6135,14 @@
         /*READ(cd->c_data, cd->c_type->ct_size)*/
         if (cd->c_type->ct_size == sizeof(char))
             return PyBytes_FromStringAndSize(cd->c_data, 1);
-#ifdef HAVE_WCHAR_H
         else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
-            assert(cd->c_type->ct_size == sizeof(wchar_t));
-            return _my_PyUnicode_FromWideChar((wchar_t *)cd->c_data, 1);
-        }
-#endif
+            switch (cd->c_type->ct_size) {
+            case 2:
+                return _my_PyUnicode_FromChar16((cffi_char16_t *)cd->c_data, 1);
+            case 4:
+                return _my_PyUnicode_FromChar32((cffi_char32_t *)cd->c_data, 1);
+            }
+        }
     }
     PyErr_Format(PyExc_TypeError, "string(): unexpected cdata '%s' argument",
                  cd->c_type->ct_name);
@@ -6120,12 +6187,14 @@
     /* byte- and unicode strings */
     ctitem = cd->c_type->ct_itemdescr;
     if (ctitem->ct_flags & CT_PRIMITIVE_CHAR) {
-        if (ctitem->ct_size == sizeof(char))
+        switch (ctitem->ct_size) {
+        case sizeof(char):
             return PyBytes_FromStringAndSize(cd->c_data, length);
-#ifdef HAVE_WCHAR_H
-        else if (ctitem->ct_size == sizeof(wchar_t))
-            return _my_PyUnicode_FromWideChar((wchar_t *)cd->c_data, length);
-#endif
+        case 2:
+            return _my_PyUnicode_FromChar16((cffi_char16_t *)cd->c_data,length);
+        case 4:
+            return _my_PyUnicode_FromChar32((cffi_char32_t *)cd->c_data,length);
+        }
     }
 
     /* else, the result is a list.  This implementation should be
@@ -6992,12 +7061,51 @@
     return PyBytes_FromStringAndSize(&x, 1);
 }
 
+/* backward-compatibility hack: instead of _cffi_to_c_char16_t() and
+ * _cffi_to_c_char32_t(), we have _cffi_to_c_wchar_t() handling whatever
+ * size is wchar_t, and _cffi_to_c_wchar3216_t() handling the opposite.
+ */
 #ifdef HAVE_WCHAR_H
-static PyObject *_cffi_from_c_wchar_t(wchar_t x) {
-    return _my_PyUnicode_FromWideChar(&x, 1);
-}
+typedef wchar_t cffi_wchar_t;
+#else
+typedef uint16_t cffi_wchar_t;   /* random pick... */
 #endif
 
+static cffi_wchar_t _cffi_to_c_wchar_t(PyObject *init)
+{
+    if (sizeof(cffi_wchar_t) == 2)
+        return (cffi_wchar_t)_convert_to_char16_t(init);
+    else
+        return (cffi_wchar_t)_convert_to_char32_t(init);
+}
+static PyObject *_cffi_from_c_wchar_t(cffi_wchar_t x) {
+    if (sizeof(cffi_wchar_t) == 2) {
+        cffi_char16_t input = x;
+        return _my_PyUnicode_FromChar16(&input, 1);
+    }
+    else {
+        cffi_char32_t input = x;
+        return _my_PyUnicode_FromChar32(&input, 1);
+    }
+}
+static int _cffi_to_c_wchar3216_t(PyObject *init)
+{
+    if (sizeof(cffi_wchar_t) == 4)
+        return (int)_convert_to_char16_t(init);
+    else
+        return (int)_convert_to_char32_t(init);
+}
+static PyObject *_cffi_from_c_wchar3216_t(int x) {
+    if (sizeof(cffi_wchar_t) == 4) {
+        cffi_char16_t input = x;
+        return _my_PyUnicode_FromChar16(&input, 1);
+    }
+    else {
+        cffi_char32_t input = x;
+        return _my_PyUnicode_FromChar32(&input, 1);
+    }
+}
+
 struct _cffi_externpy_s;      /* forward declaration */
 static void cffi_call_python(struct _cffi_externpy_s *, char *args);
 
@@ -7021,18 +7129,15 @@
     convert_to_object,
     convert_from_object,
     convert_struct_to_owning_object,
-#ifdef HAVE_WCHAR_H
-    _convert_to_wchar_t,
+    _cffi_to_c_wchar_t,
     _cffi_from_c_wchar_t,
-#else
-    0,
-    0,
-#endif
     _cffi_to_c_long_double,
     _cffi_to_c__Bool,
     _prepare_pointer_call_argument,
     convert_array_from_object,
     cffi_call_python,
+    _cffi_to_c_wchar3216_t,
+    _cffi_from_c_wchar3216_t,
 };
 
 static struct { const char *name; int value; } all_dlopen_flags[] = {
diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -2,31 +2,28 @@
  * wchar_t helpers
  */
 
-#if (Py_UNICODE_SIZE == 2) && (SIZEOF_WCHAR_T == 4)
-# define CONVERT_WCHAR_TO_SURROGATES
-#endif
+typedef uint16_t cffi_char16_t;
+typedef uint32_t cffi_char32_t;
+/* NB. cffi_char32_t is unsigned to make the logic here a bit easier */
 
 
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 2
 
 /* Before Python 2.7, PyUnicode_FromWideChar is not able to convert
    wchar_t values greater than 65535 into two-unicode-characters surrogates.
    But even the Python 2.7 version doesn't detect wchar_t values that are
    out of range(1114112), and just returns nonsense.
+
+   From cffi 1.11 we can't use it anyway, because we need a version
+   with char32_t input types.
 */
 static PyObject *
-_my_PyUnicode_FromWideChar(register const wchar_t *w,
-                           Py_ssize_t size)
+_my_PyUnicode_FromChar32(const cffi_char32_t *w, Py_ssize_t size)
 {
     PyObject *unicode;
     register Py_ssize_t i;
     Py_ssize_t alloc;
-    const wchar_t *orig_w;
-
-    if (w == NULL) {
-        PyErr_BadInternalCall();
-        return NULL;
-    }
+    const cffi_char32_t *orig_w;
 
     alloc = size;
     orig_w = w;
@@ -45,11 +42,11 @@
         register Py_UNICODE *u;
         u = PyUnicode_AS_UNICODE(unicode);
         for (i = size; i > 0; i--) {
-            if (((unsigned int)*w) > 0xFFFF) {
-                wchar_t ordinal;
-                if (((unsigned int)*w) > 0x10FFFF) {
+            if (*w > 0xFFFF) {
+                cffi_char32_t ordinal;
+                if (*w > 0x10FFFF) {
                     PyErr_Format(PyExc_ValueError,
-                                 "wchar_t out of range for "
+                                 "char32_t out of range for "
                                  "conversion to unicode: 0x%x", (int)*w);
                     Py_DECREF(unicode);
                     return NULL;
@@ -66,9 +63,53 @@
     return unicode;
 }
 
-#else
+static PyObject *
+_my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
+{
+    return PyUnicode_FromUnicode((const Py_UNICODE *)w, size);
+}
 
-# define _my_PyUnicode_FromWideChar PyUnicode_FromWideChar
+#else   /* Py_UNICODE_SIZE == 4 */
+
+static PyObject *
+_my_PyUnicode_FromChar32(const cffi_char32_t *w, Py_ssize_t size)
+{
+    return PyUnicode_FromUnicode((const Py_UNICODE *)w, size);
+}
+
+static PyObject *
+_my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
+{
+    PyObject *result = PyUnicode_FromUnicode(NULL, size);
+
+    if (result != NULL) {
+        Py_UNICODE *u_base = PyUnicode_AS_UNICODE(result);
+        Py_UNICODE *u = u_base;
+        Py_ssize_t consumed;
+
+        while (size > 0) {
+            cffi_char32_t ch = *w++;
+            size--;
+            if (0xD800 <= ch && ch <= 0xDBFF && size > 0) {
+                cffi_char32_t ch2 = *w;
+                if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+                    w++;
+                    size--;
+                }
+            }
+            *u++ = ch;
+        }
+        consumed = u - u_base;
+        if (consumed < size) {
+            if (PyUnicode_Resize(&result, consumed) < 0) {
+                Py_DECREF(result);
+                return NULL;
+            }
+        }
+    }
+    return result;
+}
 
 #endif
 
@@ -78,28 +119,70 @@
 #define AS_SURROGATE(u)   (0x10000 + (((u)[0] - 0xD800) << 10) +     \
                                      ((u)[1] - 0xDC00))
 
-static int _my_PyUnicode_AsSingleWideChar(PyObject *unicode, wchar_t *result)
+static int
+_my_PyUnicode_AsSingleChar16(PyObject *unicode, cffi_char16_t *result,
+                             char *err_got)
+{
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    if (PyUnicode_GET_SIZE(unicode) != 1) {
+        sprintf(err_got, "unicode string of length %zd",
+                PyUnicode_GET_SIZE(unicode));
+        return -1;
+    }
+#if Py_UNICODE_SIZE == 4
+    if (((unsigned int)u[0]) > 0xFFFF)
+    {
+        sprintf(err_got, "unicode character too large for 16 bits");
+        return -1;
+    }
+#endif
+    *result = (cffi_char16_t)u[0];
+    return 0;
+}
+
+static int
+_my_PyUnicode_AsSingleChar32(PyObject *unicode, cffi_char32_t *result,
+                             char *err_got)
 {
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     if (PyUnicode_GET_SIZE(unicode) == 1) {
-        *result = (wchar_t)(u[0]);
+        *result = (cffi_char32_t)u[0];
         return 0;
     }
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 2
     if (PyUnicode_GET_SIZE(unicode) == 2 && IS_SURROGATE(u)) {
         *result = AS_SURROGATE(u);
         return 0;
     }
 #endif
+    sprintf(err_got, "unicode string of length %zd",
+            PyUnicode_GET_SIZE(unicode));
     return -1;
 }
 
-static Py_ssize_t _my_PyUnicode_SizeAsWideChar(PyObject *unicode)
+static Py_ssize_t _my_PyUnicode_SizeAsChar16(PyObject *unicode)
 {
     Py_ssize_t length = PyUnicode_GET_SIZE(unicode);
     Py_ssize_t result = length;
 
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 4
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    Py_ssize_t i;
+
+    for (i=0; i<length; i++) {
+        if (u[i] > 0xFFFF)
+            result++;
+    }
+#endif
+    return result;
+}
+
+static Py_ssize_t _my_PyUnicode_SizeAsChar32(PyObject *unicode)
+{
+    Py_ssize_t length = PyUnicode_GET_SIZE(unicode);
+    Py_ssize_t result = length;
+
+#if Py_UNICODE_SIZE == 2
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     Py_ssize_t i;
 
@@ -111,15 +194,41 @@
     return result;
 }
 
-static void _my_PyUnicode_AsWideChar(PyObject *unicode,
-                                     wchar_t *result,
-                                     Py_ssize_t resultlen)
+static void _my_PyUnicode_AsChar16(PyObject *unicode,
+                                   cffi_char16_t *result,
+                                   Py_ssize_t resultlen)
+{
+    Py_ssize_t len = PyUnicode_GET_SIZE(unicode);
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    Py_ssize_t i;
+    for (i=0; i<len; i++) {
+#if Py_UNICODE_SIZE == 2
+        cffi_char16_t ordinal = u[i];
+#else
+        cffi_char32_t ordinal = u[i];
+        if (ordinal > 0xFFFF) {
+            /* NB. like CPython, ignore the problem of unicode string objects
+             * containing characters greater than sys.maxunicode.  It is
+             * easier to not add exception handling here */
+            ordinal -= 0x10000;
+            *result++ = 0xD800 | (ordinal >> 10);
+            *result++ = 0xDC00 | (ordinal & 0x3FF);
+            continue;
+        }
+#endif
+        *result++ = ordinal;
+    }
+}
+
+static void _my_PyUnicode_AsChar32(PyObject *unicode,
+                                   cffi_char32_t *result,
+                                   Py_ssize_t resultlen)
 {
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     Py_ssize_t i;
     for (i=0; i<resultlen; i++) {
-        wchar_t ordinal = *u;
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+        cffi_char32_t ordinal = *u;
+#if Py_UNICODE_SIZE == 2
         if (IS_SURROGATE(u)) {
             ordinal = AS_SURROGATE(u);
             u++;

From pypy.commits at gmail.com  Wed May 31 07:16:21 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 May 2017 04:16:21 -0700 (PDT)
Subject: [pypy-commit] pypy rawrefcount-review: move code
Message-ID: <592ea605.10d91c0a.9c30d.9897@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rawrefcount-review
Changeset: r91464:4d59821bd195
Date: 2017-05-30 11:50 +0100
http://bitbucket.org/pypy/pypy/changeset/4d59821bd195/

Log:	move code

diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -14,6 +14,49 @@
                          ('prev', lltype.Ptr(S)),
                          ('next', lltype.Ptr(S))))
 
+class RefcountSpace(GCSpace):
+    def __init__(self):
+        GCSpace.__init__(self, IncrementalMiniMarkGC, {})
+        self.trigger = []
+        self.gc.rawrefcount_init(lambda: self.trigger.append(1))
+
+    def new_rawobj(self):
+        r1 = lltype.malloc(PYOBJ_HDR, flavor='raw')
+        r1.ob_refcnt = 0
+        r1.ob_pypy_link = 0
+        return r1
+
+    def new_gcobj(self, intval, external=False):
+        saved = self.gc.nonlarge_max
+        try:
+            if external:
+                self.gc.nonlarge_max = 1
+            p1 = self.malloc(S)
+        finally:
+            self.gc.nonlarge_max = saved
+        p1.x = intval
+        return p1
+
+    def create_link(self, rawobj, gcobj, is_light=False, is_pyobj=False):
+        if is_light:
+            rawobj.ob_refcnt += REFCNT_FROM_PYPY_LIGHT
+        else:
+            rawobj.ob_refcnt += REFCNT_FROM_PYPY
+        rawaddr = llmemory.cast_ptr_to_adr(rawobj)
+        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, gcobj)
+        if is_pyobj:
+            self.gc.rawrefcount_create_link_pyobj(gcref, rawaddr)
+        else:
+            self.gc.rawrefcount_create_link_pypy(gcref, rawaddr)
+
+    def from_gc(self, gcobj):
+        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, gcobj)
+        rawaddr = self.gc.rawrefcount_from_obj(gcref)
+        if rawaddr == llmemory.NULL:
+            return None
+        else:
+            return self.gc._pyobj(rawaddr)
+
 
 class TestRawRefCount(BaseDirectGCTest):
     GCClass = IncrementalMiniMarkGC
@@ -315,49 +358,6 @@
         self._collect(major=True)
         check_alive(0)
 
-class RefcountSpace(GCSpace):
-    def __init__(self):
-        GCSpace.__init__(self, IncrementalMiniMarkGC, {})
-        self.trigger = []
-        self.gc.rawrefcount_init(lambda: self.trigger.append(1))
-
-    def new_rawobj(self):
-        r1 = lltype.malloc(PYOBJ_HDR, flavor='raw')
-        r1.ob_refcnt = 0
-        r1.ob_pypy_link = 0
-        return r1
-
-    def new_gcobj(self, intval, external=False):
-        saved = self.gc.nonlarge_max
-        try:
-            if external:
-                self.gc.nonlarge_max = 1
-            p1 = self.malloc(S)
-        finally:
-            self.gc.nonlarge_max = saved
-        p1.x = intval
-        return p1
-
-    def create_link(self, rawobj, gcobj, is_light=False, is_pyobj=False):
-        if is_light:
-            rawobj.ob_refcnt += REFCNT_FROM_PYPY_LIGHT
-        else:
-            rawobj.ob_refcnt += REFCNT_FROM_PYPY
-        rawaddr = llmemory.cast_ptr_to_adr(rawobj)
-        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, gcobj)
-        if is_pyobj:
-            self.gc.rawrefcount_create_link_pyobj(gcref, rawaddr)
-        else:
-            self.gc.rawrefcount_create_link_pypy(gcref, rawaddr)
-
-    def from_gc(self, gcobj):
-        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, gcobj)
-        rawaddr = self.gc.rawrefcount_from_obj(gcref)
-        if rawaddr == llmemory.NULL:
-            return None
-        else:
-            return self.gc._pyobj(rawaddr)
-
 from rpython.rtyper.test.test_rdict import signal_timeout, Action
 from hypothesis.strategies import (
     builds, sampled_from, binary, just, integers, text, characters, tuples,

From pypy.commits at gmail.com  Wed May 31 07:16:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 May 2017 04:16:23 -0700 (PDT)
Subject: [pypy-commit] pypy rawrefcount-review: Remove some code duplication
Message-ID: <592ea607.f4a0df0a.80337.b91a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: rawrefcount-review
Changeset: r91465:1cf3a5fddabb
Date: 2017-05-31 12:15 +0100
http://bitbucket.org/pypy/pypy/changeset/1cf3a5fddabb/

Log:	Remove some code duplication

diff --git a/rpython/memory/gc/test/test_rawrefcount.py b/rpython/memory/gc/test/test_rawrefcount.py
--- a/rpython/memory/gc/test/test_rawrefcount.py
+++ b/rpython/memory/gc/test/test_rawrefcount.py
@@ -20,8 +20,8 @@
         self.trigger = []
         self.gc.rawrefcount_init(lambda: self.trigger.append(1))
 
-    def new_rawobj(self):
-        r1 = lltype.malloc(PYOBJ_HDR, flavor='raw')
+    def new_rawobj(self, immortal=False):
+        r1 = lltype.malloc(PYOBJ_HDR, flavor='raw', immortal=immortal)
         r1.ob_refcnt = 0
         r1.ob_pypy_link = 0
         return r1
@@ -82,47 +82,20 @@
             p1.x = intval
             space.consider_constant(p1)
             return p1
-        saved = space.gc.nonlarge_max
-        try:
-            if external:
-                space.gc.nonlarge_max = 1
-            p1 = space.malloc(S)
-        finally:
-            space.gc.nonlarge_max = saved
-        p1.x = intval
+        p1 = space.new_gcobj(intval, external=external)
         if old:
             space.stackroots.append(p1)
             self._collect(major=False)
             p1 = space.stackroots.pop()
         return p1
 
-    def create_rawobj(self, immortal=False):
-        r1 = lltype.malloc(PYOBJ_HDR, flavor='raw', immortal=immortal)
-        r1.ob_refcnt = 0
-        r1.ob_pypy_link = 0
-        return r1
-
-    def create_link(self, rawobj, gcobj, is_light=False, is_pyobj=False):
-        space = self.space
-        if is_light:
-            rawobj.ob_refcnt += REFCNT_FROM_PYPY_LIGHT
-        else:
-            rawobj.ob_refcnt += REFCNT_FROM_PYPY
-        rawaddr = llmemory.cast_ptr_to_adr(rawobj)
-        gcref = lltype.cast_opaque_ptr(llmemory.GCREF, gcobj)
-        if is_pyobj:
-            assert not is_light
-            space.gc.rawrefcount_create_link_pyobj(gcref, rawaddr)
-        else:
-            space.gc.rawrefcount_create_link_pypy(gcref, rawaddr)
-
     def _rawrefcount_pair(self, intval, is_light=False, is_pyobj=False,
                           create_old=False, create_immortal=False,
                           force_external=False):
         space = self.space
         p1 = self.create_gcobj(intval, old=create_old, immortal=create_immortal, external=force_external)
-        r1 = self.create_rawobj(immortal=create_immortal)
-        self.create_link(r1, p1, is_light=is_light, is_pyobj=is_pyobj)
+        r1 = space.new_rawobj(immortal=create_immortal)
+        space.create_link(r1, p1, is_light=is_light, is_pyobj=is_pyobj)
         if is_light:
             rc = REFCNT_FROM_PYPY_LIGHT
         else:
@@ -155,7 +128,7 @@
         assert space.gc.rawrefcount_from_obj(p1ref) == r1addr
         assert space.gc.rawrefcount_to_obj(r1addr) == p1ref
         p2 = self.create_gcobj(84)
-        r2 = self.create_rawobj()
+        r2 = space.new_rawobj()
         r2.ob_refcnt += 1
         p2ref = lltype.cast_opaque_ptr(llmemory.GCREF, p2)
         r2addr = llmemory.cast_ptr_to_adr(r2)

From pypy.commits at gmail.com  Wed May 31 09:05:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 06:05:56 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: fixes
Message-ID: <592ebfb4.f4a0df0a.80337.d328@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2952:cd57d8a64e4d
Date: 2017-05-31 15:05 +0200
http://bitbucket.org/cffi/cffi/changeset/cd57d8a64e4d/

Log:	fixes

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1231,10 +1231,10 @@
     else if (PyUnicode_Check(value)) {
         /* from a unicode, we add the null terminator */
         int length;
-        if (ctitem->ct_size == 4)
+        if (ctitem->ct_size == 2)
+            length = _my_PyUnicode_SizeAsChar16(value);
+        else
             length = _my_PyUnicode_SizeAsChar32(value);
-        else
-            length = PyUnicode_GET_SIZE(value);
         return length + 1;
     }
     else {
@@ -1266,7 +1266,8 @@
 {
     /* a special case for var-sized C99 arrays */
     if ((cf->cf_type->ct_flags & CT_ARRAY) && cf->cf_type->ct_size < 0) {
-        Py_ssize_t varsizelength = get_new_array_length(&value);
+        Py_ssize_t varsizelength = get_new_array_length(
+                                      cf->cf_type->ct_itemdescr, &value);
         if (varsizelength < 0)
             return -1;
         if (optvarsize != NULL) {
@@ -1596,6 +1597,7 @@
             *(cffi_char32_t *)data = res;
             return 0;
         }
+        }
     }
     if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
 
@@ -2777,7 +2779,11 @@
     }
     else if (PyUnicode_Check(init)) {
         /* from a unicode, we add the null terminator */
-        length = _my_PyUnicode_SizeAsWideChar(init) + 1;
+        if (ctitem->ct_size == 2)
+            length = _my_PyUnicode_SizeAsChar16(init);
+        else
+            length = _my_PyUnicode_SizeAsChar32(init);
+        length += 1;
     }
     else if ((ctitem->ct_flags & CT_IS_FILE) && PyFile_Check(init)) {
         *output_data = (char *)PyFile_AsFile(init);
@@ -3503,7 +3509,7 @@
         dataoffset = offsetof(CDataObject_own_nolength, alignment);
         datasize = ct->ct_size;
         if (datasize < 0) {
-            explicitlength = get_new_array_length(&init);
+            explicitlength = get_new_array_length(ct->ct_itemdescr, &init);
             if (explicitlength < 0)
                 return NULL;
             ctitem = ct->ct_itemdescr;
@@ -6121,6 +6127,7 @@
                 }
                 return _my_PyUnicode_FromChar32(start, length);
             }
+            }
         }
     }
     else if (cd->c_type->ct_flags & CT_IS_ENUM) {
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2098,22 +2098,36 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+
+def test_char16():
+    assert sizeof(new_primitive_type("char16_t")) == 2
+    _test_wchar_variant("char16_t")
+
+def test_char32():
+    assert sizeof(new_primitive_type("char32_t")) == 4
+    _test_wchar_variant("char32_t")
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2128,9 +2142,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2165,17 +2179,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2183,8 +2197,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2211,7 +2225,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)

From pypy.commits at gmail.com  Wed May 31 09:47:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 06:47:15 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: Tests and fixes
Message-ID: <592ec963.88da1c0a.c7b8.6622@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2953:ac2fc8661954
Date: 2017-05-31 15:47 +0200
http://bitbucket.org/cffi/cffi/changeset/ac2fc8661954/

Log:	Tests and fixes

diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -80,29 +80,31 @@
 static PyObject *
 _my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
 {
+    /* 'size' is the length of the 'w' array */
     PyObject *result = PyUnicode_FromUnicode(NULL, size);
 
     if (result != NULL) {
         Py_UNICODE *u_base = PyUnicode_AS_UNICODE(result);
         Py_UNICODE *u = u_base;
-        Py_ssize_t consumed;
 
-        while (size > 0) {
-            cffi_char32_t ch = *w++;
-            size--;
-            if (0xD800 <= ch && ch <= 0xDBFF && size > 0) {
-                cffi_char32_t ch2 = *w;
-                if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
-                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
-                    w++;
-                    size--;
+        if (size == 1) {      /* performance only */
+            *u = (cffi_char32_t)*w;
+        }
+        else {
+            while (size > 0) {
+                cffi_char32_t ch = *w++;
+                size--;
+                if (0xD800 <= ch && ch <= 0xDBFF && size > 0) {
+                    cffi_char32_t ch2 = *w;
+                    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
+                        ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+                        w++;
+                        size--;
+                    }
                 }
+                *u++ = ch;
             }
-            *u++ = ch;
-        }
-        consumed = u - u_base;
-        if (consumed < size) {
-            if (PyUnicode_Resize(&result, consumed) < 0) {
+            if (PyUnicode_Resize(&result, u - u_base) < 0) {
                 Py_DECREF(result);
                 return NULL;
             }
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -1,6 +1,7 @@
 import py, sys, platform
 import pytest
 from testing.cffi0 import backend_tests, test_function, test_ownlib
+from testing.support import u
 from cffi import FFI
 import _cffi_backend
 
@@ -397,6 +398,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -508,3 +511,38 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+    def test_explicitly_defined_char16_t(self):
+        ffi = FFI()
+        ffi.cdef("typedef uint16_t char16_t;")
+        x = ffi.cast("char16_t", 1234)
+        assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+    def test_char16_t(self):
+        ffi = FFI()
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        ffi = FFI()
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars string
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2250,3 +2250,9 @@
     int f(int a) { return a + 40; }
     """, extra_compile_args=['-fvisibility=hidden'])
     assert lib.f(2) == 42
+
+def test_override_default_definition():
+    ffi = FFI()
+    ffi.cdef("typedef long int16_t, char16_t;")
+    lib = verify(ffi, "test_override_default_definition", "")
+    assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")

From pypy.commits at gmail.com  Wed May 31 10:05:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 07:05:33 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: more tests
Message-ID: <592ecdad.505c1c0a.edbbb.5e33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2954:5c4f518e8d62
Date: 2017-05-31 16:05 +0200
http://bitbucket.org/cffi/cffi/changeset/5c4f518e8d62/

Log:	more tests

diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -2,6 +2,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
+from testing.support import u
 
 
 SOURCE = """\
@@ -92,6 +93,15 @@
 }
 
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+    return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+    return (unsigned int)(a + 42);
+}
 """
 
 class TestOwnLib(object):
@@ -300,3 +310,18 @@
         pfn = ffi.addressof(lib, "test_getting_errno")
         assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
         assert pfn == lib.test_getting_errno
+
+    def test_char16_char32_t(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            char16_t foo_2bytes(char16_t);
+            char32_t foo_4bytes(char32_t);
+        """)
+        lib = ffi.dlopen(self.module)
+        assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -241,7 +241,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true
@@ -384,6 +384,10 @@
     lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
     assert lib.foo(uniexample1) == uniexample2
 
+def test_char16_char32_type():
+    py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+                 "working in ffi.verify() mode")
+
 def test_no_argument():
     ffi = FFI()
     ffi.cdef("int foo(void);")
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1672,6 +1672,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -221,7 +221,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true

From pypy.commits at gmail.com  Wed May 31 10:42:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 07:42:25 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: in-progress
Message-ID: <592ed651.d4151c0a.1937e.ce9e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2955:983e7f434f13
Date: 2017-05-31 16:42 +0200
http://bitbucket.org/cffi/cffi/changeset/983e7f434f13/

Log:	in-progress

diff --git a/c/cffi1_module.c b/c/cffi1_module.c
--- a/c/cffi1_module.c
+++ b/c/cffi1_module.c
@@ -2,8 +2,9 @@
 #include "parse_c_type.c"
 #include "realize_c_type.c"
 
-#define CFFI_VERSION_MIN    0x2601
-#define CFFI_VERSION_MAX    0x27FF
+#define CFFI_VERSION_MIN            0x2601
+#define CFFI_VERSION_CHAR16CHAR32   0x2801
+#define CFFI_VERSION_MAX            0x28FF
 
 typedef struct FFIObject_s FFIObject;
 typedef struct LibObject_s LibObject;
@@ -183,6 +184,8 @@
     num_exports = 25;
     if (ctx->flags & 1)    /* set to mean that 'extern "Python"' is used */
         num_exports = 26;
+    if (version >= CFFI_VERSION_CHAR16CHAR32)
+        num_exports = 28;
     memcpy(exports, (char *)cffi_exports, num_exports * sizeof(void *));
 
     /* make the module object */
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -153,6 +153,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
     };
     PyObject *x;
 
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -62,11 +62,16 @@
     typedef unsigned char _Bool;
 #  endif
 # endif
+# if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
+    typedef uint16_t char16_t;
+    typedef int32_t char32_t;
+# endif
 #else
 # include <stdint.h>
 # if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
+# include <uchar.h>
 #endif
 
 #ifdef __GNUC__
@@ -174,7 +179,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +224,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_to_c_wchar_t(o);
+    else
+        return _cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_to_c_wchar_t(o);
+    else
+        return _cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1744,3 +1744,30 @@
         exec("from _test_import_from_lib import *", d)
         assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
                 ['ffi', 'lib'])
+
+    def test_char16_t(self):
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -24,10 +24,11 @@
     assert ''.join(map(str, recomp.cffi_types)) == expected_output
 
 def verify(ffi, module_name, source, *args, **kwds):
+    no_cpp = kwds.pop('no_cpp', False)
     kwds.setdefault('undef_macros', ['NDEBUG'])
     module_name = '_CFFI_' + module_name
     ffi.set_source(module_name, source)
-    if not os.environ.get('NO_CPP'):     # test the .cpp mode too
+    if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
     else:
@@ -2256,3 +2257,20 @@
     ffi.cdef("typedef long int16_t, char16_t;")
     lib = verify(ffi, "test_override_default_definition", "")
     assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+    ffi = FFI()
+    ffi.cdef("""
+        char16_t foo_2bytes(char16_t);
+        char32_t foo_4bytes(char32_t);
+    """)
+    lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+    char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+    """, no_cpp=no_cpp)
+    assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+
+def test_char16_char32_plain_c():
+    test_char16_char32_type(no_cpp=True)

From pypy.commits at gmail.com  Wed May 31 10:57:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 07:57:36 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: fixes
Message-ID: <592ed9e0.08891c0a.c8e37.b93e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2956:2c0195e0f232
Date: 2017-05-31 16:57 +0200
http://bitbucket.org/cffi/cffi/changeset/2c0195e0f232/

Log:	fixes

diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -493,6 +493,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -501,6 +502,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -134,7 +134,7 @@
 #if Py_UNICODE_SIZE == 4
     if (((unsigned int)u[0]) > 0xFFFF)
     {
-        sprintf(err_got, "unicode character too large for 16 bits");
+        sprintf(err_got, "larger-than-0xFFFF character");
         return -1;
     }
 #endif
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -542,7 +542,12 @@
         y = ffi.new("char32_t[]", u+'\u1234\u5678')
         assert len(y) == 3
         assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
-        z = ffi.new("char32_t[]", u+'\U00012345')
+        py_uni = u+'\U00012345'
+        z = ffi.new("char32_t[]", py_uni)
         assert len(z) == 2
-        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars string
-        assert ffi.string(z) == u+'\U00012345'
+        assert list(z) == [py_uni, u+'\x00']    # maybe a 2-unichars string
+        assert ffi.string(z) == py_uni
+        if len(py_uni) == 1:    # 4-bytes unicodes in Python
+            s = ffi.new("char32_t[]", u+'\ud808\udf00')
+            assert len(s) == 3
+            assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2271,6 +2271,7 @@
     assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
     assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
     assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+    py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
 
 def test_char16_char32_plain_c():
     test_char16_char32_type(no_cpp=True)

From pypy.commits at gmail.com  Wed May 31 11:17:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 08:17:07 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: tweaks
Message-ID: <592ede73.03541c0a.3b56f.2cc8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2957:8e7afeadc985
Date: 2017-05-31 17:16 +0200
http://bitbucket.org/cffi/cffi/changeset/8e7afeadc985/

Log:	tweaks

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -164,9 +164,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -231,15 +231,15 @@
 typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
 #endif
 
-_CFFI_UNUSED_FN static int _cffi_to_c_char16_t(PyObject *o)
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
 {
     if (sizeof(_cffi_wchar_t) == 2)
-        return _cffi_to_c_wchar_t(o);
+        return (uint16_t)_cffi_to_c_wchar_t(o);
     else
-        return _cffi_to_c_wchar3216_t(o);
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
 }
 
-_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(int x)
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
 {
     if (sizeof(_cffi_wchar_t) == 2)
         return _cffi_from_c_wchar_t(x);
@@ -250,9 +250,9 @@
 _CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
 {
     if (sizeof(_cffi_wchar_t) == 4)
-        return _cffi_to_c_wchar_t(o);
+        return (int)_cffi_to_c_wchar_t(o);
     else
-        return _cffi_to_c_wchar3216_t(o);
+        return (int)_cffi_to_c_wchar3216_t(o);
 }
 
 _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2272,6 +2272,8 @@
     assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
     assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
     py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+    py.test.raises(TypeError, lib.foo_2bytes, 1234)
+    py.test.raises(TypeError, lib.foo_4bytes, 1234)
 
 def test_char16_char32_plain_c():
     test_char16_char32_type(no_cpp=True)

From pypy.commits at gmail.com  Wed May 31 12:16:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 09:16:07 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: Implement complexes,
 enough to make the test_c part pass
Message-ID: <592eec47.4c3a1c0a.7e266.dfdb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91466:593f66be444b
Date: 2017-05-31 18:15 +0200
http://bitbucket.org/pypy/pypy/changeset/593f66be444b/

Log:	Implement complexes, enough to make the test_c part pass

diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,6 +91,11 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
+    def complex(self):
+        with self as ptr:
+            w_result = self.ctype.complex(ptr)
+        return w_result
+
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -405,6 +410,13 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
+    def write_raw_complex_data(self, real, imag):
+        with self as ptr:
+            halfsize = self.ctype.size >> 1
+            ptr2 = rffi.ptradd(ptr, halfsize)
+            misc.write_raw_float_data(ptr, real, halfsize)
+            misc.write_raw_float_data(ptr2, imag, halfsize)
+
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -646,6 +658,7 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
+    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,18 +212,21 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    space = self.space
-    if self.size < 0:
-        raise oefmt(space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
+
+def _notimplemented_ffi_type(self, is_result_type, extra=''):
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s",
-                self.name, self.size, place)
+    raise oefmt(self.space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s%s",
+                self.name, self.size, place, extra)
+
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    if self.size < 0:
+        raise oefmt(self.space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
+    raise _notimplemented_ffi_type(self, is_result_type)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -260,6 +263,13 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
+def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
+    raise _notimplemented_ffi_type(self, is_result_type,
+        extra = " (the support for complex types inside libffi "
+                "is mostly missing at this point, so CFFI only "
+                "supports complex types as arguments or return "
+                "value in API-mode functions)")
+
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -276,6 +286,7 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
+W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,6 +73,14 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
+    def complex(self, cdata):
+        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+        # calling complex(), just like <cdata 'int'> cannot be directly
+        # converted by calling float()
+        space = self.space
+        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
+                    self.name)
+
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,3 +532,51 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
+
+
+class W_CTypePrimitiveComplex(W_CTypePrimitive):
+    _attrs_ = []
+
+    def cast(self, w_ob):
+        space = self.space
+        if isinstance(w_ob, cdataobj.W_CData):
+            if not isinstance(w_ob.ctype, W_CTypePrimitive):
+                raise oefmt(space.w_TypeError,
+                            "cannot cast ctype '%s' to ctype '%s'",
+                            w_ob.ctype.name, self.name)
+            w_ob = w_ob.convert_to_object()
+        #
+        imag = 0.0
+        if space.isinstance_w(w_ob, space.w_bytes):
+            real = self.cast_str(w_ob)
+        elif space.isinstance_w(w_ob, space.w_unicode):
+            real = self.cast_unicode(w_ob)
+        else:
+            real, imag = space.unpackcomplex(w_ob)
+        w_cdata = cdataobj.W_CDataMem(space, self)
+        w_cdata.write_raw_complex_data(real, imag)
+        return w_cdata
+
+    def complex(self, cdata):
+        return self.convert_to_object(cdata)
+
+    def convert_to_object(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        real = misc.read_raw_float_data(cdata, halfsize)
+        imag = misc.read_raw_float_data(cdata2, halfsize)
+        return self.space.newcomplex(real, imag)
+
+    def convert_from_object(self, cdata, w_ob):
+        space = self.space
+        real, imag = space.unpackcomplex(w_ob)
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        misc.write_raw_float_data(cdata,  real, halfsize)
+        misc.write_raw_float_data(cdata2, imag, halfsize)
+
+    def nonzero(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        return (misc.is_nonnull_float(cdata, halfsize) |
+                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
+def eptype(name, TYPE, ctypecls, rep=1):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,6 +94,9 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
+eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
+eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
+
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -174,37 +174,56 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
-        assert bool(cast(p, 0))
+        p = new_primitive_type(name + " _Complex")
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
-        assert cast(p, -1.1j) != cast(p, -1.1j)
+        assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1105,6 +1124,34 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
+def test_call_function_25():
+    BDouble = new_primitive_type("double")
+    BDoubleComplex = new_primitive_type("double _Complex")
+    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")

From pypy.commits at gmail.com  Wed May 31 12:22:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 09:22:36 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: parse_c_type
Message-ID: <592eedcc.899adf0a.997e.04dd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91467:01d27eabba7c
Date: 2017-05-31 18:22 +0200
http://bitbucket.org/pypy/pypy/changeset/01d27eabba7c/

Log:	parse_c_type

diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,8 +105,10 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,6 +171,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -613,6 +614,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -668,6 +670,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -678,6 +682,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -731,9 +736,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -800,6 +807,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok, "_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,8 +78,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,6 +148,8 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
+            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
+            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -273,6 +275,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize

From pypy.commits at gmail.com  Wed May 31 12:26:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 09:26:25 -0700 (PDT)
Subject: [pypy-commit] cffi default: no-op,
 but avoids confusion because these two are always listed in the
Message-ID: <592eeeb1.0a8f1c0a.84b0d.b788@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2958:28290fb948f1
Date: 2017-05-31 18:26 +0200
http://bitbucket.org/cffi/cffi/changeset/28290fb948f1/

Log:	no-op, but avoids confusion because these two are always listed in
	the opposite order elsewhere

diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1704,8 +1704,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
             'double _Complex',
-            'float _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)

From pypy.commits at gmail.com  Wed May 31 12:39:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 09:39:17 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the version to 1.11
Message-ID: <592ef1b5.38a6df0a.12763.7e90@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2959:1e353c3fe102
Date: 2017-05-31 18:39 +0200
http://bitbucket.org/cffi/cffi/changeset/1e353c3fe102/

Log:	Update the version to 1.11

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.10.0"
+#define CFFI_VERSION  "1.11.0"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -45,9 +45,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.10'
+version = '1.11'
 # The full version, including alpha/beta/rc tags.
-release = '1.10.0'
+release = '1.11.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.10.0.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.11.0.tar.gz
 
-   - MD5: 2b5fa41182ed0edaf929a789e602a070
+   - MD5: ...
 
-   - SHA: 8484aba03d1e64367d3110c0e36c1ed052b43f12
+   - SHA: ...
 
-   - SHA256: b3b02911eb1f6ada203b0763ba924234629b51586f72a21faacc638269f4ced5
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -183,7 +183,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.10.0',
+        version='1.11.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h']}

From pypy.commits at gmail.com  Wed May 31 13:12:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 10:12:41 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: Call support for functions with
 complex args/return types
Message-ID: <592ef989.9199df0a.fedc3.30eb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91468:92d597a61c6a
Date: 2017-05-31 19:02 +0200
http://bitbucket.org/pypy/pypy/changeset/92d597a61c6a/

Log:	Call support for functions with complex args/return types

diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,6 +8,7 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
+from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -70,6 +71,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -209,7 +212,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.
+        # trampoline functions for PyPy.  (Same with complex numbers.)
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -218,11 +221,13 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
+                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
+                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
+            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
+                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,6 +1819,68 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
+    def test_function_returns_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float _Complex f1(float a, float b);",
+            "test_function_returns_float_complex", """
+            #include <complex.h>
+            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+    def test_function_returns_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double _Complex f1(double a, double b);",
+            "test_function_returns_double_complex", """
+            #include <complex.h>
+            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert result.imag == 2*5.1  # exact
+
+    def test_function_argument_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float f1(float _Complex x);",
+            "test_function_argument_float_complex", """
+            #include <complex.h>
+            static float f1(float _Complex x) { return cabsf(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-5
+        result2 = lib.f1(ffi.cast("float _Complex", x))
+        assert result2 == result
+
+    def test_function_argument_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double f1(double _Complex);",
+            "test_function_argument_double_complex", """
+            #include <complex.h>
+            static double f1(double _Complex x) { return cabs(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-11
+        result2 = lib.f1(ffi.cast("double _Complex", x))
+        assert result2 == result
+
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,6 +8,7 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -83,8 +84,9 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
-                return w_result_cdata.structobj
+                ctyperesptr = w_result_cdata.ctype
+                assert isinstance(ctyperesptr, W_CTypePointer)
+                return w_result_cdata._do_getitem(ctyperesptr, 0)
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -109,13 +111,14 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
+    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union>
+        farg = fargs[i]      # <ptr to struct/union/complex>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make

From pypy.commits at gmail.com  Wed May 31 13:12:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 10:12:43 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: Update the version number here
Message-ID: <592ef98b.f188df0a.5259d.b86b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91469:d795c4cf2cd0
Date: 2017-05-31 19:12 +0200
http://bitbucket.org/pypy/pypy/changeset/d795c4cf2cd0/

Log:	Update the version number here

diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.11.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:

From pypy.commits at gmail.com  Wed May 31 13:17:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 10:17:34 -0700 (PDT)
Subject: [pypy-commit] cffi default: update
Message-ID: <592efaae.43861c0a.ef183.19a0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2960:e4e863b06bc7
Date: 2017-05-31 19:17 +0200
http://bitbucket.org/cffi/cffi/changeset/e4e863b06bc7/

Log:	update

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -3854,7 +3854,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))

From pypy.commits at gmail.com  Wed May 31 13:18:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 10:18:01 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: update
Message-ID: <592efac9.50841c0a.1ad7c.5cb1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91470:866666ccccd6
Date: 2017-05-31 19:17 +0200
http://bitbucket.org/pypy/pypy/changeset/866666ccccd6/

Log:	update

diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3843,7 +3843,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))

From pypy.commits at gmail.com  Wed May 31 13:39:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 May 2017 10:39:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Update _testcapimodule.c to 3.5.3,
 disabling things that don't compile
Message-ID: <592effdf.899adf0a.997e.3068@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91471:252218fff126
Date: 2017-05-31 17:52 +0100
http://bitbucket.org/pypy/pypy/changeset/252218fff126/

Log:	Update _testcapimodule.c to 3.5.3, disabling things that don't
	compile

diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -1,5 +1,5 @@
-/* Verbatim copy of Modules/_testcapimodule.c from CPython 3.3,
- * except that "run_in_subinterp" is commented out
+/* Copy of Modules/_testcapimodule.c from CPython 3.3,
+ * with things that PyPy doesn't support disabled.
  */
 /*
  * C Extension module to test Python interpreter C APIs.
@@ -14,6 +14,12 @@
 #include <float.h>
 #include "structmember.h"
 #include "datetime.h"
+// #include "marshal.h"
+#include <signal.h>
+
+#ifdef MS_WINDOWS
+#  include <winsock2.h>         /* struct timeval */
+#endif
 
 #ifdef WITH_THREAD
 #include "pythread.h"
@@ -68,6 +74,76 @@
 }
 
 static PyObject*
+test_sizeof_c_types(PyObject *self)
+{
+#if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wtype-limits"
+#endif
+#define CHECK_SIZEOF(TYPE, EXPECTED)         \
+    if (EXPECTED != sizeof(TYPE))  {         \
+        PyErr_Format(TestError,              \
+            "sizeof(%s) = %u instead of %u", \
+            #TYPE, sizeof(TYPE), EXPECTED);  \
+        return (PyObject*)NULL;              \
+    }
+#define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0)
+#define CHECK_SIGNNESS(TYPE, SIGNED)         \
+    if (IS_SIGNED(TYPE) != SIGNED) {         \
+        PyErr_Format(TestError,              \
+            "%s signness is, instead of %i",  \
+            #TYPE, IS_SIGNED(TYPE), SIGNED); \
+        return (PyObject*)NULL;              \
+    }
+
+    /* integer types */
+    CHECK_SIZEOF(Py_UCS1, 1);
+    CHECK_SIZEOF(Py_UCS2, 2);
+    CHECK_SIZEOF(Py_UCS4, 4);
+    CHECK_SIGNNESS(Py_UCS1, 0);
+    CHECK_SIGNNESS(Py_UCS2, 0);
+    CHECK_SIGNNESS(Py_UCS4, 0);
+#ifdef HAVE_INT32_T
+    CHECK_SIZEOF(PY_INT32_T, 4);
+    CHECK_SIGNNESS(PY_INT32_T, 1);
+#endif
+#ifdef HAVE_UINT32_T
+    CHECK_SIZEOF(PY_UINT32_T, 4);
+    CHECK_SIGNNESS(PY_UINT32_T, 0);
+#endif
+#ifdef HAVE_INT64_T
+    CHECK_SIZEOF(PY_INT64_T, 8);
+    CHECK_SIGNNESS(PY_INT64_T, 1);
+#endif
+#ifdef HAVE_UINT64_T
+    CHECK_SIZEOF(PY_UINT64_T, 8);
+    CHECK_SIGNNESS(PY_UINT64_T, 0);
+#endif
+
+    /* pointer/size types */
+    CHECK_SIZEOF(size_t, sizeof(void *));
+    CHECK_SIGNNESS(size_t, 0);
+    CHECK_SIZEOF(Py_ssize_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_ssize_t, 1);
+
+    CHECK_SIZEOF(Py_uintptr_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_uintptr_t, 0);
+    CHECK_SIZEOF(Py_intptr_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_intptr_t, 1);
+
+    Py_INCREF(Py_None);
+    return Py_None;
+
+#undef IS_SIGNED
+#undef CHECK_SIGNESS
+#undef CHECK_SIZEOF
+#if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+#pragma GCC diagnostic pop
+#endif
+}
+
+
+static PyObject*
 test_list_api(PyObject *self)
 {
     PyObject* list;
@@ -175,6 +251,17 @@
     return Py_None;
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject*
+dict_hassplittable(PyObject *self, PyObject *arg)
+{
+    if (!PyArg_Parse(arg, "O!:dict_hassplittable", &PyDict_Type, &arg)) {
+        return NULL;
+    }
+    return PyBool_FromLong(_PyDict_HasSplitTable((PyDictObject*)arg));
+}
+#endif  /* PYPY_VERSION */
 
 /* Issue #4701: Check that PyObject_Hash implicitly calls
  *   PyType_Ready if it hasn't already been called
@@ -799,6 +886,120 @@
 
 #endif  /* ifdef HAVE_LONG_LONG */
 
+static PyObject *
+return_none(void *unused)
+{
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+raise_error(void *unused)
+{
+    PyErr_SetNone(PyExc_ValueError);
+    return NULL;
+}
+
+static int
+test_buildvalue_N_error(const char *fmt)
+{
+    PyObject *arg, *res;
+
+    arg = PyList_New(0);
+    if (arg == NULL) {
+        return -1;
+    }
+
+    Py_INCREF(arg);
+    res = Py_BuildValue(fmt, return_none, NULL, arg);
+    if (res == NULL) {
+        return -1;
+    }
+    Py_DECREF(res);
+    if (Py_REFCNT(arg) != 1) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "arg was not decrefed in successful "
+                     "Py_BuildValue(\"%s\")", fmt);
+        return -1;
+    }
+
+    Py_INCREF(arg);
+    res = Py_BuildValue(fmt, raise_error, NULL, arg);
+    if (res != NULL || !PyErr_Occurred()) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "Py_BuildValue(\"%s\") didn't complain", fmt);
+        return -1;
+    }
+    PyErr_Clear();
+    if (Py_REFCNT(arg) != 1) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "arg was not decrefed in failed "
+                     "Py_BuildValue(\"%s\")", fmt);
+        return -1;
+    }
+    Py_DECREF(arg);
+    return 0;
+}
+
+static PyObject *
+test_buildvalue_N(PyObject *self, PyObject *noargs)
+{
+    PyObject *arg, *res;
+
+    arg = PyList_New(0);
+    if (arg == NULL) {
+        return NULL;
+    }
+    Py_INCREF(arg);
+    res = Py_BuildValue("N", arg);
+    if (res == NULL) {
+        return NULL;
+    }
+    if (res != arg) {
+        return raiseTestError("test_buildvalue_N",
+                              "Py_BuildValue(\"N\") returned wrong result");
+    }
+    if (Py_REFCNT(arg) != 2) {
+        return raiseTestError("test_buildvalue_N",
+                              "arg was not decrefed in Py_BuildValue(\"N\")");
+    }
+    Py_DECREF(res);
+    Py_DECREF(arg);
+
+    if (test_buildvalue_N_error("O&N") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("(O&N)") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("[O&N]") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("{O&N}") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("{()O&(())N}") < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+get_args(PyObject *self, PyObject *args)
+{
+    if (args == NULL) {
+        args = Py_None;
+    }
+    Py_INCREF(args);
+    return args;
+}
+
+static PyObject *
+get_kwargs(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    if (kwargs == NULL) {
+        kwargs = Py_None;
+    }
+    Py_INCREF(kwargs);
+    return kwargs;
+}
+
 /* Test tuple argument processing */
 static PyObject *
 getargs_tuple(PyObject *self, PyObject *args)
@@ -1010,12 +1211,78 @@
 }
 
 static PyObject *
+getargs_f(PyObject *self, PyObject *args)
+{
+    float f;
+    if (!PyArg_ParseTuple(args, "f", &f))
+        return NULL;
+    return PyFloat_FromDouble(f);
+}
+
+static PyObject *
+getargs_d(PyObject *self, PyObject *args)
+{
+    double d;
+    if (!PyArg_ParseTuple(args, "d", &d))
+        return NULL;
+    return PyFloat_FromDouble(d);
+}
+
+static PyObject *
+getargs_D(PyObject *self, PyObject *args)
+{
+    Py_complex cval;
+    if (!PyArg_ParseTuple(args, "D", &cval))
+        return NULL;
+    return PyComplex_FromCComplex(cval);
+}
+
+static PyObject *
+getargs_S(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "S", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
+getargs_Y(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "Y", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
+getargs_U(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "U", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
 getargs_c(PyObject *self, PyObject *args)
 {
     char c;
     if (!PyArg_ParseTuple(args, "c", &c))
         return NULL;
-    return PyBytes_FromStringAndSize(&c, 1);
+    return PyLong_FromLong((unsigned char)c);
+}
+
+static PyObject *
+getargs_C(PyObject *self, PyObject *args)
+{
+    int c;
+    if (!PyArg_ParseTuple(args, "C", &c))
+        return NULL;
+    return PyLong_FromLong(c);
 }
 
 static PyObject *
@@ -1170,6 +1437,84 @@
         Py_RETURN_NONE;
 }
 
+static PyObject *
+getargs_es(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    char *str;
+
+    if (!PyArg_ParseTuple(args, "O|s", &arg, &encoding))
+        return NULL;
+    if (!PyArg_Parse(arg, "es", encoding, &str))
+        return NULL;
+    result = PyBytes_FromString(str);
+    PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_et(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    char *str;
+
+    if (!PyArg_ParseTuple(args, "O|s", &arg, &encoding))
+        return NULL;
+    if (!PyArg_Parse(arg, "et", encoding, &str))
+        return NULL;
+    result = PyBytes_FromString(str);
+    PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_es_hash(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    PyByteArrayObject *buffer = NULL;
+    char *str = NULL;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "O|sY", &arg, &encoding, &buffer))
+        return NULL;
+    if (buffer != NULL) {
+        str = PyByteArray_AS_STRING(buffer);
+        size = PyByteArray_GET_SIZE(buffer);
+    }
+    if (!PyArg_Parse(arg, "es#", encoding, &str, &size))
+        return NULL;
+    result = PyBytes_FromStringAndSize(str, size);
+    if (buffer == NULL)
+        PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_et_hash(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    PyByteArrayObject *buffer = NULL;
+    char *str = NULL;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "O|sY", &arg, &encoding, &buffer))
+        return NULL;
+    if (buffer != NULL) {
+        str = PyByteArray_AS_STRING(buffer);
+        size = PyByteArray_GET_SIZE(buffer);
+    }
+    if (!PyArg_Parse(arg, "et#", encoding, &str, &size))
+        return NULL;
+    result = PyBytes_FromStringAndSize(str, size);
+    if (buffer == NULL)
+        PyMem_Free(str);
+    return result;
+}
+
 /* Test the s and z codes for PyArg_ParseTuple.
 */
 static PyObject *
@@ -1427,6 +1772,20 @@
     else
         return raiseTestError("test_widechar",
                               "PyUnicode_FromUnicode(L\"\\U00110000\", 1) didn't fail");
+
+    wide = PyUnicode_FromUnicode(NULL, 1);
+    if (wide == NULL)
+        return NULL;
+    PyUnicode_AS_UNICODE(wide)[0] = invalid[0];
+    if (_PyUnicode_Ready(wide) < 0) {
+        Py_DECREF(wide);
+        PyErr_Clear();
+    }
+    else {
+        Py_DECREF(wide);
+        return raiseTestError("test_widechar",
+                              "PyUnicode_Ready() didn't fail");
+    }
 #endif
 
     Py_RETURN_NONE;
@@ -1441,7 +1800,7 @@
 
     if (!PyArg_ParseTuple(args, "Un", &unicode, &buflen))
         return NULL;
-    buffer = PyMem_Malloc(buflen * sizeof(wchar_t));
+    buffer = PyMem_New(wchar_t, buflen);
     if (buffer == NULL)
         return PyErr_NoMemory();
 
@@ -1484,6 +1843,72 @@
     return Py_BuildValue("(Nn)", result, size);
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject *
+unicode_asucs4(PyObject *self, PyObject *args)
+{
+    PyObject *unicode, *result;
+    Py_UCS4 *buffer;
+    int copy_null;
+    Py_ssize_t str_len, buf_len;
+
+    if (!PyArg_ParseTuple(args, "Unp:unicode_asucs4", &unicode, &str_len, &copy_null)) {
+        return NULL;
+    }
+
+    buf_len = str_len + 1;
+    buffer = PyMem_NEW(Py_UCS4, buf_len);
+    if (buffer == NULL) {
+        return PyErr_NoMemory();
+    }
+    memset(buffer, 0, sizeof(Py_UCS4)*buf_len);
+    buffer[str_len] = 0xffffU;
+
+    if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) {
+        PyMem_FREE(buffer);
+        return NULL;
+    }
+
+    result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len);
+    PyMem_FREE(buffer);
+    return result;
+}
+
+static PyObject *
+unicode_copycharacters(PyObject *self, PyObject *args)
+{
+    PyObject *from, *to, *to_copy;
+    Py_ssize_t from_start, to_start, how_many, copied;
+
+    if (!PyArg_ParseTuple(args, "UnOnn:unicode_copycharacters", &to, &to_start,
+                          &from, &from_start, &how_many)) {
+        return NULL;
+    }
+
+    if (PyUnicode_READY(to) < 0) {
+        return NULL;
+    }
+
+    if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to),
+                                  PyUnicode_MAX_CHAR_VALUE(to)))) {
+        return NULL;
+    }
+    if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) {
+        Py_DECREF(to_copy);
+        return NULL;
+    }
+
+    if ((copied = PyUnicode_CopyCharacters(to_copy, to_start, from,
+                                           from_start, how_many)) < 0) {
+        Py_DECREF(to_copy);
+        return NULL;
+    }
+
+    return Py_BuildValue("(Nn)", to_copy, copied);
+}
+#endif  /* PYPY_VERSION */
+
 static PyObject *
 unicode_encodedecimal(PyObject *self, PyObject *args)
 {
@@ -1644,7 +2069,7 @@
              {-0xffffL, 16, -1},
              {0xfffffffL, 28, 1},
              {-0xfffffffL, 28, -1}};
-    int i;
+    size_t i;
 
     for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) {
         size_t nbits;
@@ -1709,6 +2134,18 @@
 }
 
 static PyObject *
+set_errno(PyObject *self, PyObject *args)
+{
+    int new_errno;
+
+    if (!PyArg_ParseTuple(args, "i:set_errno", &new_errno))
+        return NULL;
+
+    errno = new_errno;
+    Py_RETURN_NONE;
+}
+
+static PyObject *
 test_set_exc_info(PyObject *self, PyObject *args)
 {
     PyObject *orig_exc;
@@ -1879,6 +2316,8 @@
 }
 #endif
 
+#ifndef PYPY_VERSION
+
 /* Some tests of PyUnicode_FromFormat().  This needs more tests. */
 static PyObject *
 test_string_from_format(PyObject *self, PyObject *args)
@@ -1890,7 +2329,7 @@
     result = PyUnicode_FromFormat(FORMAT, (TYPE)1);                 \
     if (result == NULL)                                             \
         return NULL;                                                \
-    if (PyUnicode_CompareWithASCIIString(result, "1")) {     \
+    if (!_PyUnicode_EqualToASCIIString(result, "1")) {              \
         msg = FORMAT " failed at 1";                                \
         goto Fail;                                                  \
     }                                                               \
@@ -1920,6 +2359,7 @@
 
 #undef CHECK_1_FORMAT
 }
+#endif  /* PYPY_VERSION */
 
 
 static PyObject *
@@ -2394,6 +2834,111 @@
     return PyMemoryView_FromBuffer(&info);
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_from_contiguous(PyObject* self, PyObject *noargs)
+{
+    int data[9] = {-1,-1,-1,-1,-1,-1,-1,-1,-1};
+    int init[5] = {0, 1, 2, 3, 4};
+    Py_ssize_t itemsize = sizeof(int);
+    Py_ssize_t shape = 5;
+    Py_ssize_t strides = 2 * itemsize;
+    Py_buffer view = {
+        data,
+        NULL,
+        5 * itemsize,
+        itemsize,
+        1,
+        1,
+        NULL,
+        &shape,
+        &strides,
+        NULL,
+        NULL
+    };
+    int *ptr;
+    int i;
+
+    PyBuffer_FromContiguous(&view, init, view.len, 'C');
+    ptr = view.buf;
+    for (i = 0; i < 5; i++) {
+        if (ptr[2*i] != i) {
+            PyErr_SetString(TestError,
+                "test_from_contiguous: incorrect result");
+            return NULL;
+        }
+    }
+
+    view.buf = &data[8];
+    view.strides[0] = -2 * itemsize;
+
+    PyBuffer_FromContiguous(&view, init, view.len, 'C');
+    ptr = view.buf;
+    for (i = 0; i < 5; i++) {
+        if (*(ptr-2*i) != i) {
+            PyErr_SetString(TestError,
+                "test_from_contiguous: incorrect result");
+            return NULL;
+        }
+    }
+
+    Py_RETURN_NONE;
+}
+#endif  /* PYPY_VERSION */
+
+#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION)
+extern PyTypeObject _PyBytesIOBuffer_Type;
+
+static PyObject *
+test_pep3118_obsolete_write_locks(PyObject* self, PyObject *noargs)
+{
+    PyTypeObject *type = &_PyBytesIOBuffer_Type;
+    PyObject *b;
+    char *dummy[1];
+    int ret, match;
+
+    /* PyBuffer_FillInfo() */
+    ret = PyBuffer_FillInfo(NULL, NULL, dummy, 1, 0, PyBUF_SIMPLE);
+    match = PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_BufferError);
+    PyErr_Clear();
+    if (ret != -1 || match == 0)
+        goto error;
+
+    /* bytesiobuf_getbuffer() */
+    b = type->tp_alloc(type, 0);
+    if (b == NULL) {
+        return NULL;
+    }
+
+    ret = PyObject_GetBuffer(b, NULL, PyBUF_SIMPLE);
+    Py_DECREF(b);
+    match = PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_BufferError);
+    PyErr_Clear();
+    if (ret != -1 || match == 0)
+        goto error;
+
+    Py_RETURN_NONE;
+
+error:
+    PyErr_SetString(TestError,
+        "test_pep3118_obsolete_write_locks: failure");
+    return NULL;
+}
+#endif
+
+/* This tests functions that historically supported write locks.  It is
+   wrong to call getbuffer() with view==NULL and a compliant getbufferproc
+   is entitled to segfault in that case. */
+static PyObject *
+getbuffer_with_null_view(PyObject* self, PyObject *obj)
+{
+    if (PyObject_GetBuffer(obj, NULL, PyBUF_SIMPLE) < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
 /* Test that the fatal error from not having a current thread doesn't
    cause an infinite loop.  Run via Lib/test/test_capi.py */
 static PyObject *
@@ -2444,14 +2989,27 @@
     return PyLong_FromLong(r);
 }
 
+static int
+check_time_rounding(int round)
+{
+    if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) {
+        PyErr_SetString(PyExc_ValueError, "invalid rounding");
+        return -1;
+    }
+    return 0;
+}
+
 static PyObject *
 test_pytime_object_to_time_t(PyObject *self, PyObject *args)
 {
     PyObject *obj;
     time_t sec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_time_t", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTime_t(obj, &sec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_time_t", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1)
         return NULL;
     return _PyLong_FromTime_t(sec);
 }
@@ -2462,9 +3020,12 @@
     PyObject *obj;
     time_t sec;
     long usec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_timeval", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTimeval(obj, &sec, &usec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timeval", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1)
         return NULL;
     return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec);
 }
@@ -2475,15 +3036,444 @@
     PyObject *obj;
     time_t sec;
     long nsec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_timespec", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTimespec(obj, &sec, &nsec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timespec", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1)
         return NULL;
     return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec);
 }
 
+static void
+slot_tp_del(PyObject *self)
+{
+    _Py_IDENTIFIER(__tp_del__);
+    PyObject *del, *res;
+    PyObject *error_type, *error_value, *error_traceback;
+
+    /* Temporarily resurrect the object. */
+    assert(self->ob_refcnt == 0);
+    self->ob_refcnt = 1;
+
+    /* Save the current exception, if any. */
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+    /* Execute __del__ method, if any. */
+    del = _PyObject_LookupSpecial(self, &PyId___tp_del__);
+    if (del != NULL) {
+        res = PyEval_CallObject(del, NULL);
+        if (res == NULL)
+            PyErr_WriteUnraisable(del);
+        else
+            Py_DECREF(res);
+        Py_DECREF(del);
+    }
+
+    /* Restore the saved exception. */
+    PyErr_Restore(error_type, error_value, error_traceback);
+
+    /* Undo the temporary resurrection; can't use DECREF here, it would
+     * cause a recursive call.
+     */
+    assert(self->ob_refcnt > 0);
+    if (--self->ob_refcnt == 0)
+        return;         /* this is the normal path out */
+
+    /* __del__ resurrected it!  Make it look like the original Py_DECREF
+     * never happened.
+     */
+    {
+        Py_ssize_t refcnt = self->ob_refcnt;
+        _Py_NewReference(self);
+        self->ob_refcnt = refcnt;
+    }
+    assert(!PyType_IS_GC(Py_TYPE(self)) ||
+           _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+    /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+     * we need to undo that. */
+    _Py_DEC_REFTOTAL;
+    /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+     * chain, so no more to do there.
+     * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+     * _Py_NewReference bumped tp_allocs:  both of those need to be
+     * undone.
+     */
+#ifdef COUNT_ALLOCS
+    --Py_TYPE(self)->tp_frees;
+    --Py_TYPE(self)->tp_allocs;
+#endif
+}
+
+static PyObject *
+with_tp_del(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    PyTypeObject *tp;
+
+    if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj))
+        return NULL;
+    tp = (PyTypeObject *) obj;
+    if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
+        PyErr_Format(PyExc_TypeError,
+                     "heap type expected, got %R", obj);
+        return NULL;
+    }
+    tp->tp_del = slot_tp_del;
+    Py_INCREF(obj);
+    return obj;
+}
+
 #endif  /* PYPY_VERSION */
 
+static PyMethodDef ml;
+
+static PyObject *
+create_cfunction(PyObject *self, PyObject *args)
+{
+    return PyCFunction_NewEx(&ml, self, NULL);
+}
+
+static PyMethodDef ml = {
+    "create_cfunction",
+    create_cfunction,
+    METH_NOARGS,
+    NULL
+};
+
+static PyObject *
+_test_incref(PyObject *ob)
+{
+    Py_INCREF(ob);
+    return ob;
+}
+
+static PyObject *
+test_xincref_doesnt_leak(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_XINCREF(_test_incref(obj));
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_incref_doesnt_leak(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_INCREF(_test_incref(obj));
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_xdecref_doesnt_leak(PyObject *ob)
+{
+    Py_XDECREF(PyLong_FromLong(0));
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_decref_doesnt_leak(PyObject *ob)
+{
+    Py_DECREF(PyLong_FromLong(0));
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_incref_decref_API(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_IncRef(obj);
+    Py_DecRef(obj);
+    Py_DecRef(obj);
+    Py_RETURN_NONE;
+}
+
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_pymem_alloc0(PyObject *self)
+{
+    void *ptr;
+
+    ptr = PyMem_RawMalloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
+    ptr = PyMem_RawCalloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawCalloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
+    ptr = PyMem_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyMem_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyObject_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    ptr = PyObject_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    Py_RETURN_NONE;
+}
+
+typedef struct {
+    PyMemAllocatorEx alloc;
+
+    size_t malloc_size;
+    size_t calloc_nelem;
+    size_t calloc_elsize;
+    void *realloc_ptr;
+    size_t realloc_new_size;
+    void *free_ptr;
+} alloc_hook_t;
+
+static void* hook_malloc (void* ctx, size_t size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->malloc_size = size;
+    return hook->alloc.malloc(hook->alloc.ctx, size);
+}
+
+static void* hook_calloc (void* ctx, size_t nelem, size_t elsize)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->calloc_nelem = nelem;
+    hook->calloc_elsize = elsize;
+    return hook->alloc.calloc(hook->alloc.ctx, nelem, elsize);
+}
+
+static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->realloc_ptr = ptr;
+    hook->realloc_new_size = new_size;
+    return hook->alloc.realloc(hook->alloc.ctx, ptr, new_size);
+}
+
+static void hook_free (void *ctx, void *ptr)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->free_ptr = ptr;
+    hook->alloc.free(hook->alloc.ctx, ptr);
+}
+
+static PyObject *
+test_setallocators(PyMemAllocatorDomain domain)
+{
+    PyObject *res = NULL;
+    const char *error_msg;
+    alloc_hook_t hook;
+    PyMemAllocatorEx alloc;
+    size_t size, size2, nelem, elsize;
+    void *ptr, *ptr2;
+
+    memset(&hook, 0, sizeof(hook));
+
+    alloc.ctx = &hook;
+    alloc.malloc = &hook_malloc;
+    alloc.calloc = &hook_calloc;
+    alloc.realloc = &hook_realloc;
+    alloc.free = &hook_free;
+    PyMem_GetAllocator(domain, &hook.alloc);
+    PyMem_SetAllocator(domain, &alloc);
+
+    size = 42;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawMalloc(size); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Malloc(size); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Malloc(size); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "malloc failed";
+        goto fail;
+    }
+
+    if (hook.malloc_size != size) {
+        error_msg = "malloc invalid size";
+        goto fail;
+    }
+
+    size2 = 200;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr2 = PyMem_RawRealloc(ptr, size2); break;
+    case PYMEM_DOMAIN_MEM: ptr2 = PyMem_Realloc(ptr, size2); break;
+    case PYMEM_DOMAIN_OBJ: ptr2 = PyObject_Realloc(ptr, size2); break;
+    default: ptr2 = NULL; break;
+    }
+
+    if (ptr2 == NULL) {
+        error_msg = "realloc failed";
+        goto fail;
+    }
+
+    if (hook.realloc_ptr != ptr
+        || hook.realloc_new_size != size2) {
+        error_msg = "realloc invalid parameters";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr2); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr2); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr2); break;
+    }
+
+    if (hook.free_ptr != ptr2) {
+        error_msg = "free invalid pointer";
+        goto fail;
+    }
+
+    nelem = 2;
+    elsize = 5;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawCalloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Calloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Calloc(nelem, elsize); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "calloc failed";
+        goto fail;
+    }
+
+    if (hook.calloc_nelem != nelem || hook.calloc_elsize != elsize) {
+        error_msg = "calloc invalid nelem or elsize";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr); break;
+    }
+
+    Py_INCREF(Py_None);
+    res = Py_None;
+    goto finally;
+
+fail:
+    PyErr_SetString(PyExc_RuntimeError, error_msg);
+
+finally:
+    PyMem_SetAllocator(domain, &hook.alloc);
+    return res;
+}
+
+static PyObject *
+test_pymem_setrawallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_RAW);
+}
+
+static PyObject *
+test_pymem_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_MEM);
+}
+
+static PyObject *
+test_pyobject_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_OBJ);
+}
+
+#endif  /* PYPY_VERSION */
+
+PyDoc_STRVAR(docstring_empty,
+""
+);
+
+PyDoc_STRVAR(docstring_no_signature,
+"This docstring has no signature."
+);
+
+PyDoc_STRVAR(docstring_with_invalid_signature,
+"docstring_with_invalid_signature($module, /, boo)\n"
+"\n"
+"This docstring has an invalid signature."
+);
+
+PyDoc_STRVAR(docstring_with_invalid_signature2,
+"docstring_with_invalid_signature2($module, /, boo)\n"
+"\n"
+"--\n"
+"\n"
+"This docstring also has an invalid signature."
+);
+
+PyDoc_STRVAR(docstring_with_signature,
+"docstring_with_signature($module, /, sig)\n"
+"--\n"
+"\n"
+"This docstring has a valid signature."
+);
+
+PyDoc_STRVAR(docstring_with_signature_but_no_doc,
+"docstring_with_signature_but_no_doc($module, /, sig)\n"
+"--\n"
+"\n"
+);
+
+PyDoc_STRVAR(docstring_with_signature_and_extra_newlines,
+"docstring_with_signature_and_extra_newlines($module, /, parameter)\n"
+"--\n"
+"\n"
+"\n"
+"This docstring has a valid signature and some extra newlines."
+);
+
+PyDoc_STRVAR(docstring_with_signature_with_defaults,
+"docstring_with_signature_with_defaults(module, s='avocado',\n"
+"        b=b'bytes', d=3.14, i=35, n=None, t=True, f=False,\n"
+"        local=the_number_three, sys=sys.maxsize,\n"
+"        exp=sys.maxsize - 1)\n"
+"--\n"
+"\n"
+"\n"
+"\n"
+"This docstring has a valid signature with parameters,\n"
+"and the parameters take defaults of varying types."
+);
+
 #ifdef WITH_THREAD
 typedef struct {
     PyThread_type_lock start_event;
@@ -2573,16 +3563,349 @@
 }
 #endif   /* WITH_THREAD */
 
+static PyObject*
+test_raise_signal(PyObject* self, PyObject *args)
+{
+    int signum, err;
+
+    if (PyArg_ParseTuple(args, "i:raise_signal", &signum) < 0)
+        return NULL;
+
+    err = raise(signum);
+    if (err)
+        return PyErr_SetFromErrno(PyExc_OSError);
+
+    if (PyErr_CheckSignals() < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
+/* marshal */
+
+#ifndef PYPY_VERSION
+
+static PyObject*
+pymarshal_write_long_to_file(PyObject* self, PyObject *args)
+{
+    long value;
+    char *filename;
+    int version;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "lsi:pymarshal_write_long_to_file",
+                          &value, &filename, &version))
+        return NULL;
+
+    fp = fopen(filename, "wb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    PyMarshal_WriteLongToFile(value, fp, version);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+pymarshal_write_object_to_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    char *filename;
+    int version;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "Osi:pymarshal_write_object_to_file",
+                          &obj, &filename, &version))
+        return NULL;
+
+    fp = fopen(filename, "wb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    PyMarshal_WriteObjectToFile(obj, fp, version);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+pymarshal_read_short_from_file(PyObject* self, PyObject *args)
+{
+    int value;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_short_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    value = PyMarshal_ReadShortFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    return Py_BuildValue("il", value, pos);
+}
+
+static PyObject*
+pymarshal_read_long_from_file(PyObject* self, PyObject *args)
+{
+    long value, pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_long_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    value = PyMarshal_ReadLongFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    return Py_BuildValue("ll", value, pos);
+}
+
+static PyObject*
+pymarshal_read_last_object_from_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_last_object_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    obj = PyMarshal_ReadLastObjectFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    return Py_BuildValue("Nl", obj, pos);
+}
+
+static PyObject*
+pymarshal_read_object_from_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_object_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    obj = PyMarshal_ReadObjectFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    return Py_BuildValue("Nl", obj, pos);
+}
+#endif  /* PYPY_VERSION */
+
+static PyObject*
+return_null_without_error(PyObject *self, PyObject *args)
+{
+    /* invalid call: return NULL without setting an error,
+     * _Py_CheckFunctionResult() must detect such bug at runtime. */
+    PyErr_Clear();
+    return NULL;
+}
+
+static PyObject*
+return_result_with_error(PyObject *self, PyObject *args)
+{
+    /* invalid call: return a result with an error set,
+     * _Py_CheckFunctionResult() must detect such bug at runtime. */
+    PyErr_SetNone(PyExc_ValueError);
+    Py_RETURN_NONE;
+}
+
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_pytime_fromseconds(PyObject *self, PyObject *args)
+{
+    int seconds;
+    _PyTime_t ts;
+
+    if (!PyArg_ParseTuple(args, "i", &seconds))
+        return NULL;
+    ts = _PyTime_FromSeconds(seconds);
+    return _PyTime_AsNanosecondsObject(ts);
+}
+
+static PyObject *
+test_pytime_fromsecondsobject(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    int round;
+    _PyTime_t ts;
+
+    if (!PyArg_ParseTuple(args, "Oi", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_FromSecondsObject(&ts, obj, round) == -1)
+        return NULL;
+    return _PyTime_AsNanosecondsObject(ts);
+}
+
+static PyObject *
+test_pytime_assecondsdouble(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    _PyTime_t ts;
+    double d;
+
+    if (!PyArg_ParseTuple(args, "L", &ns))
+        return NULL;
+    ts = _PyTime_FromNanoseconds(ns);
+    d = _PyTime_AsSecondsDouble(ts);
+    return PyFloat_FromDouble(d);
+}
+
+static PyObject *
+test_PyTime_AsTimeval(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t;
+    struct timeval tv;
+    PyObject *seconds;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    if (_PyTime_AsTimeval(t, &tv, round) < 0)
+        return NULL;
+
+    seconds = PyLong_FromLong((PY_LONG_LONG)tv.tv_sec);
+    if (seconds == NULL)
+        return NULL;
+    return Py_BuildValue("Nl", seconds, tv.tv_usec);
+}
+
+#ifdef HAVE_CLOCK_GETTIME
+static PyObject *
+test_PyTime_AsTimespec(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    _PyTime_t t;
+    struct timespec ts;
+
+    if (!PyArg_ParseTuple(args, "L", &ns))
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    if (_PyTime_AsTimespec(t, &ts) == -1)
+        return NULL;
+    return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec);
+}
+#endif
+
+static PyObject *
+test_PyTime_AsMilliseconds(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t, ms;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    ms = _PyTime_AsMilliseconds(t, round);
+    /* This conversion rely on the fact that _PyTime_t is a number of
+       nanoseconds */
+    return _PyTime_AsNanosecondsObject(ms);
+}
+
+static PyObject *
+test_PyTime_AsMicroseconds(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t, ms;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    ms = _PyTime_AsMicroseconds(t, round);
+    /* This conversion rely on the fact that _PyTime_t is a number of
+       nanoseconds */
+    return _PyTime_AsNanosecondsObject(ms);
+}
+
+
+static PyObject*
+get_recursion_depth(PyObject *self, PyObject *args)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+
+    /* subtract one to ignore the frame of the get_recursion_depth() call */
+    return PyLong_FromLong(tstate->recursion_depth - 1);
+}
+
+#endif  /* PYPY_VERSION */
 
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
+    {"set_errno",               set_errno,                       METH_VARARGS},
     {"test_config",             (PyCFunction)test_config,        METH_NOARGS},
+    {"test_sizeof_c_types",     (PyCFunction)test_sizeof_c_types, METH_NOARGS},
     {"test_datetime_capi",  test_datetime_capi,              METH_NOARGS},
     {"test_list_api",           (PyCFunction)test_list_api,      METH_NOARGS},
     {"test_dict_iteration",     (PyCFunction)test_dict_iteration,METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"dict_hassplittable",      dict_hassplittable,              METH_O},
+#endif
     {"test_lazy_hash_inheritance",      (PyCFunction)test_lazy_hash_inheritance,METH_NOARGS},
     {"test_long_api",           (PyCFunction)test_long_api,      METH_NOARGS},
+    {"test_xincref_doesnt_leak",(PyCFunction)test_xincref_doesnt_leak,      METH_NOARGS},
+    {"test_incref_doesnt_leak", (PyCFunction)test_incref_doesnt_leak,      METH_NOARGS},
+    {"test_xdecref_doesnt_leak",(PyCFunction)test_xdecref_doesnt_leak,      METH_NOARGS},
+    {"test_decref_doesnt_leak", (PyCFunction)test_decref_doesnt_leak,      METH_NOARGS},
+    {"test_incref_decref_API",  (PyCFunction)test_incref_decref_API,       METH_NOARGS},
     {"test_long_and_overflow", (PyCFunction)test_long_and_overflow,
      METH_NOARGS},
     {"test_long_as_double",     (PyCFunction)test_long_as_double,METH_NOARGS},
@@ -2592,12 +3915,24 @@
     {"test_empty_argparse", (PyCFunction)test_empty_argparse,METH_NOARGS},
     {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS},
     {"test_null_strings",       (PyCFunction)test_null_strings,  METH_NOARGS},
+#ifndef PYPY_VERSION
     {"test_string_from_format", (PyCFunction)test_string_from_format, METH_NOARGS},
+#endif
     {"test_with_docstring", (PyCFunction)test_with_docstring, METH_NOARGS,
      PyDoc_STR("This is a pretty normal docstring.")},
     {"test_string_to_double", (PyCFunction)test_string_to_double, METH_NOARGS},
     {"test_unicode_compare_with_ascii", (PyCFunction)test_unicode_compare_with_ascii, METH_NOARGS},
     {"test_capsule", (PyCFunction)test_capsule, METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"test_from_contiguous", (PyCFunction)test_from_contiguous, METH_NOARGS},
+#endif
+#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION)
+    {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS},
+#endif
+    {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O},
+    {"test_buildvalue_N",       test_buildvalue_N,               METH_NOARGS},
+    {"get_args", get_args, METH_VARARGS},
+    {"get_kwargs", (PyCFunction)get_kwargs, METH_VARARGS|METH_KEYWORDS},
     {"getargs_tuple",           getargs_tuple,                   METH_VARARGS},
     {"getargs_keywords", (PyCFunction)getargs_keywords,
       METH_VARARGS|METH_KEYWORDS},
@@ -2621,7 +3956,14 @@
         (PyCFunction)test_long_long_and_overflow, METH_NOARGS},
     {"test_L_code",             (PyCFunction)test_L_code,        METH_NOARGS},
 #endif
+    {"getargs_f",               getargs_f,                       METH_VARARGS},
+    {"getargs_d",               getargs_d,                       METH_VARARGS},
+    {"getargs_D",               getargs_D,                       METH_VARARGS},
+    {"getargs_S",               getargs_S,                       METH_VARARGS},
+    {"getargs_Y",               getargs_Y,                       METH_VARARGS},
+    {"getargs_U",               getargs_U,                       METH_VARARGS},
     {"getargs_c",               getargs_c,                       METH_VARARGS},
+    {"getargs_C",               getargs_C,                       METH_VARARGS},
     {"getargs_s",               getargs_s,                       METH_VARARGS},
     {"getargs_s_star",          getargs_s_star,                  METH_VARARGS},
     {"getargs_s_hash",          getargs_s_hash,                  METH_VARARGS},
@@ -2636,6 +3978,10 @@
     {"getargs_Z",               getargs_Z,                       METH_VARARGS},
     {"getargs_Z_hash",          getargs_Z_hash,                  METH_VARARGS},
     {"getargs_w_star",          getargs_w_star,                  METH_VARARGS},
+    {"getargs_es",              getargs_es,                      METH_VARARGS},
+    {"getargs_et",              getargs_et,                      METH_VARARGS},
+    {"getargs_es_hash",         getargs_es_hash,                 METH_VARARGS},
+    {"getargs_et_hash",         getargs_et_hash,                 METH_VARARGS},
     {"codec_incrementalencoder",
      (PyCFunction)codec_incrementalencoder,                      METH_VARARGS},
     {"codec_incrementaldecoder",
@@ -2646,6 +3992,10 @@
     {"test_widechar",           (PyCFunction)test_widechar,      METH_NOARGS},
     {"unicode_aswidechar",      unicode_aswidechar,              METH_VARARGS},
     {"unicode_aswidecharstring",unicode_aswidecharstring,        METH_VARARGS},
+#ifndef PYPY_VERSION
+    {"unicode_asucs4",          unicode_asucs4,                  METH_VARARGS},
+    {"unicode_copycharacters",  unicode_copycharacters,          METH_VARARGS},
+#endif
     {"unicode_encodedecimal",   unicode_encodedecimal,           METH_VARARGS},
     {"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},
     {"unicode_legacy_string",   unicode_legacy_string,           METH_VARARGS},
@@ -2671,11 +4021,81 @@
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
+    {"with_tp_del",             with_tp_del,                     METH_VARARGS},
 #endif
+    {"create_cfunction",        create_cfunction,                METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"test_pymem_alloc0",
+     (PyCFunction)test_pymem_alloc0, METH_NOARGS},
+    {"test_pymem_setrawallocators",
+     (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
+    {"test_pymem_setallocators",
+     (PyCFunction)test_pymem_setallocators, METH_NOARGS},
+    {"test_pyobject_setallocators",
+     (PyCFunction)test_pyobject_setallocators, METH_NOARGS},
+#endif
+    {"no_docstring",
+        (PyCFunction)test_with_docstring, METH_NOARGS},
+    {"docstring_empty",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_empty},
+    {"docstring_no_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_no_signature},
+    {"docstring_with_invalid_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_invalid_signature},
+    {"docstring_with_invalid_signature2",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_invalid_signature2},
+    {"docstring_with_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature},
+    {"docstring_with_signature_but_no_doc",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_but_no_doc},
+    {"docstring_with_signature_and_extra_newlines",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_and_extra_newlines},
+    {"docstring_with_signature_with_defaults",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_with_defaults},
+    {"raise_signal",
+     (PyCFunction)test_raise_signal, METH_VARARGS},
 #ifdef WITH_THREAD
     {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_O,
      PyDoc_STR("set_error_class(error_class) -> None")},
 #endif
+#ifndef PYPY_VERSION
+    {"pymarshal_write_long_to_file",
+        pymarshal_write_long_to_file, METH_VARARGS},
+    {"pymarshal_write_object_to_file",
+        pymarshal_write_object_to_file, METH_VARARGS},
+    {"pymarshal_read_short_from_file",
+        pymarshal_read_short_from_file, METH_VARARGS},
+    {"pymarshal_read_long_from_file",
+        pymarshal_read_long_from_file, METH_VARARGS},
+    {"pymarshal_read_last_object_from_file",
+        pymarshal_read_last_object_from_file, METH_VARARGS},
+    {"pymarshal_read_object_from_file",
+        pymarshal_read_object_from_file, METH_VARARGS},
+#endif
+    {"return_null_without_error",
+        return_null_without_error, METH_NOARGS},
+    {"return_result_with_error",
+        return_result_with_error, METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"PyTime_FromSeconds", test_pytime_fromseconds,  METH_VARARGS},
+    {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject,  METH_VARARGS},
+    {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS},
+    {"PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS},
+#ifdef HAVE_CLOCK_GETTIME
+    {"PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS},
+#endif
+    {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
+    {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
+    {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
+#endif
     {NULL, NULL} /* sentinel */
 };
 
@@ -2835,6 +4255,201 @@
 };
 
 
+typedef struct {
+    PyObject_HEAD
+} matmulObject;
+
+static PyObject *
+matmulType_matmul(PyObject *self, PyObject *other)
+{
+    return Py_BuildValue("(sOO)", "matmul", self, other);
+}
+
+static PyObject *
+matmulType_imatmul(PyObject *self, PyObject *other)
+{
+    return Py_BuildValue("(sOO)", "imatmul", self, other);
+}
+
+static void
+matmulType_dealloc(PyObject *self)
+{
+    Py_TYPE(self)->tp_free(self);
+}
+
+static PyNumberMethods matmulType_as_number = {
+    0,                          /* nb_add */
+    0,                          /* nb_subtract */
+    0,                          /* nb_multiply */
+    0,                          /* nb_remainde r*/
+    0,                          /* nb_divmod */
+    0,                          /* nb_power */
+    0,                          /* nb_negative */
+    0,                          /* tp_positive */
+    0,                          /* tp_absolute */
+    0,                          /* tp_bool */
+    0,                          /* nb_invert */
+    0,                          /* nb_lshift */
+    0,                          /* nb_rshift */
+    0,                          /* nb_and */
+    0,                          /* nb_xor */
+    0,                          /* nb_or */
+    0,                          /* nb_int */
+    0,                          /* nb_reserved */
+    0,                          /* nb_float */
+    0,                          /* nb_inplace_add */
+    0,                          /* nb_inplace_subtract */
+    0,                          /* nb_inplace_multiply */
+    0,                          /* nb_inplace_remainder */
+    0,                          /* nb_inplace_power */
+    0,                          /* nb_inplace_lshift */
+    0,                          /* nb_inplace_rshift */
+    0,                          /* nb_inplace_and */
+    0,                          /* nb_inplace_xor */
+    0,                          /* nb_inplace_or */
+    0,                          /* nb_floor_divide */
+    0,                          /* nb_true_divide */
+    0,                          /* nb_inplace_floor_divide */
+    0,                          /* nb_inplace_true_divide */
+    0,                          /* nb_index */
+    matmulType_matmul,        /* nb_matrix_multiply */
+    matmulType_imatmul        /* nb_matrix_inplace_multiply */
+};
+
+static PyTypeObject matmulType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "matmulType",
+    sizeof(matmulObject),               /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    matmulType_dealloc,                 /* destructor tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_reserved */
+    0,                                  /* tp_repr */
+    &matmulType_as_number,              /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    PyObject_GenericGetAttr,            /* tp_getattro */
+    PyObject_GenericSetAttr,            /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    0,                                  /* tp_flags */
+    "C level type with matrix operations defined",
+    0,                                  /* traverseproc tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    PyType_GenericNew,                  /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *ao_iterator;
+} awaitObject;
+
+
+static PyObject *
+awaitObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyObject *v;
+    awaitObject *ao;
+
+    if (!PyArg_UnpackTuple(args, "awaitObject", 1, 1, &v))
+        return NULL;
+
+    ao = (awaitObject *)type->tp_alloc(type, 0);
+    if (ao == NULL) {
+        return NULL;
+    }
+
+    Py_INCREF(v);
+    ao->ao_iterator = v;
+
+    return (PyObject *)ao;
+}
+
+
+static void
+awaitObject_dealloc(awaitObject *ao)
+{
+    Py_CLEAR(ao->ao_iterator);
+    Py_TYPE(ao)->tp_free(ao);
+}
+
+
+static PyObject *
+awaitObject_await(awaitObject *ao)
+{
+    Py_INCREF(ao->ao_iterator);
+    return ao->ao_iterator;
+}
+
+static PyAsyncMethods awaitType_as_async = {
+    (unaryfunc)awaitObject_await,           /* am_await */
+    0,                                      /* am_aiter */
+    0                                       /* am_anext */
+};
+
+
+static PyTypeObject awaitType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "awaitType",
+    sizeof(awaitObject),                /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    (destructor)awaitObject_dealloc,    /* destructor tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    &awaitType_as_async,                /* tp_as_async */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    PyObject_GenericGetAttr,            /* tp_getattro */
+    PyObject_GenericSetAttr,            /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    0,                                  /* tp_flags */
+    "C level type with tp_as_async",
+    0,                                  /* traverseproc tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    awaitObject_new,                    /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
 
 static struct PyModuleDef _testcapimodule = {
     PyModuleDef_HEAD_INIT,
@@ -2848,6 +4463,9 @@
     NULL
 };
 
+/* Per PEP 489, this module will not be converted to multi-phase initialization
+ */
+
 PyMODINIT_FUNC
 PyInit__testcapi(void)
 {
@@ -2856,7 +4474,7 @@
     m = PyModule_Create(&_testcapimodule);
     if (m == NULL)
         return NULL;
-    
+
     Py_TYPE(&_HashInheritanceTester_Type)=&PyType_Type;
 
     Py_TYPE(&test_structmembersType)=&PyType_Type;
@@ -2864,6 +4482,15 @@
     /* don't use a name starting with "test", since we don't want
        test_capi to automatically call this */
     PyModule_AddObject(m, "_test_structmembersType", (PyObject *)&test_structmembersType);
+    if (PyType_Ready(&matmulType) < 0)
+        return NULL;
+    Py_INCREF(&matmulType);
+    PyModule_AddObject(m, "matmulType", (PyObject *)&matmulType);
+
+    if (PyType_Ready(&awaitType) < 0)
+        return NULL;
+    Py_INCREF(&awaitType);
+    PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);
 
     PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
     PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
@@ -2890,6 +4517,8 @@
     Py_INCREF(&PyInstanceMethod_Type);
     PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type);
 
+    PyModule_AddIntConstant(m, "the_number_three", 3);
+
     TestError = PyErr_NewException("_testcapi.error", NULL, NULL);
     Py_INCREF(TestError);
     PyModule_AddObject(m, "error", TestError);

From pypy.commits at gmail.com  Wed May 31 13:49:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 May 2017 10:49:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip crashing test
Message-ID: <592f021d.412f1c0a.462e6.aa10@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91472:2420f0a96b7c
Date: 2017-05-31 18:49 +0100
http://bitbucket.org/pypy/pypy/changeset/2420f0a96b7c/

Log:	Skip crashing test

diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py
--- a/lib-python/3/test/test_capi.py
+++ b/lib-python/3/test/test_capi.py
@@ -26,6 +26,11 @@
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
 Py_DEBUG = hasattr(sys, 'gettotalrefcount')
 
+skips = []
+if support.check_impl_detail(pypy=True):
+    skips += [
+            'test_widechar',
+            ]
 
 def testfunction(self):
     """some doc"""
@@ -558,7 +563,7 @@
 class Test_testcapi(unittest.TestCase):
     def test__testcapi(self):
         for name in dir(_testcapi):
-            if name.startswith('test_'):
+            if name.startswith('test_') and name not in skips:
                 with self.subTest("internal", name=name):
                     test = getattr(_testcapi, name)
                     test()

From pypy.commits at gmail.com  Wed May 31 14:26:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 11:26:37 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: import cffi/e4e863b06bc7
Message-ID: <592f0add.d2991c0a.12164.da3a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91473:97086aa395d9
Date: 2017-05-31 19:26 +0100
http://bitbucket.org/pypy/pypy/changeset/97086aa395d9/

Log:	import cffi/e4e863b06bc7

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -734,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1187,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1479,6 +1486,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,15 +240,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,6 +1705,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
+            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,6 +156,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -281,6 +283,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,7 +48,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,7 +1553,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2001,6 +2002,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,15 +220,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Wed May 31 14:30:05 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 11:30:05 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: Document branch
Message-ID: <592f0bad.0c9adf0a.13516.9f1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91474:960fd87cce17
Date: 2017-05-31 19:28 +0100
http://bitbucket.org/pypy/pypy/changeset/960fd87cce17/

Log:	Document branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,6 @@
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8
 
+.. branch: cffi-complex
 
+Part of the upgrade to cffi 1.11

From pypy.commits at gmail.com  Wed May 31 14:30:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 11:30:07 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: ready to merge
Message-ID: <592f0baf.56691c0a.9218d.acf2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91475:fe3d4678c68f
Date: 2017-05-31 19:28 +0100
http://bitbucket.org/pypy/pypy/changeset/fe3d4678c68f/

Log:	ready to merge


From pypy.commits at gmail.com  Wed May 31 14:30:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 31 May 2017 11:30:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge cffi-complex
Message-ID: <592f0bb2.eb8edf0a.460a.204a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91476:5e7bef99e9f5
Date: 2017-05-31 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/5e7bef99e9f5/

Log:	hg merge cffi-complex

	"float _Complex", "double _Complex"

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -734,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1187,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1479,6 +1486,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,6 @@
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8
 
+.. branch: cffi-complex
 
+Part of the upgrade to cffi 1.11
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.11.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,6 +91,11 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
+    def complex(self):
+        with self as ptr:
+            w_result = self.ctype.complex(ptr)
+        return w_result
+
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -405,6 +410,13 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
+    def write_raw_complex_data(self, real, imag):
+        with self as ptr:
+            halfsize = self.ctype.size >> 1
+            ptr2 = rffi.ptradd(ptr, halfsize)
+            misc.write_raw_float_data(ptr, real, halfsize)
+            misc.write_raw_float_data(ptr2, imag, halfsize)
+
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -646,6 +658,7 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
+    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,8 +105,10 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,18 +212,21 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    space = self.space
-    if self.size < 0:
-        raise oefmt(space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
+
+def _notimplemented_ffi_type(self, is_result_type, extra=''):
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s",
-                self.name, self.size, place)
+    raise oefmt(self.space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s%s",
+                self.name, self.size, place, extra)
+
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    if self.size < 0:
+        raise oefmt(self.space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
+    raise _notimplemented_ffi_type(self, is_result_type)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -260,6 +263,13 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
+def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
+    raise _notimplemented_ffi_type(self, is_result_type,
+        extra = " (the support for complex types inside libffi "
+                "is mostly missing at this point, so CFFI only "
+                "supports complex types as arguments or return "
+                "value in API-mode functions)")
+
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -276,6 +286,7 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
+W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,6 +73,14 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
+    def complex(self, cdata):
+        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+        # calling complex(), just like <cdata 'int'> cannot be directly
+        # converted by calling float()
+        space = self.space
+        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
+                    self.name)
+
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,3 +532,51 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
+
+
+class W_CTypePrimitiveComplex(W_CTypePrimitive):
+    _attrs_ = []
+
+    def cast(self, w_ob):
+        space = self.space
+        if isinstance(w_ob, cdataobj.W_CData):
+            if not isinstance(w_ob.ctype, W_CTypePrimitive):
+                raise oefmt(space.w_TypeError,
+                            "cannot cast ctype '%s' to ctype '%s'",
+                            w_ob.ctype.name, self.name)
+            w_ob = w_ob.convert_to_object()
+        #
+        imag = 0.0
+        if space.isinstance_w(w_ob, space.w_bytes):
+            real = self.cast_str(w_ob)
+        elif space.isinstance_w(w_ob, space.w_unicode):
+            real = self.cast_unicode(w_ob)
+        else:
+            real, imag = space.unpackcomplex(w_ob)
+        w_cdata = cdataobj.W_CDataMem(space, self)
+        w_cdata.write_raw_complex_data(real, imag)
+        return w_cdata
+
+    def complex(self, cdata):
+        return self.convert_to_object(cdata)
+
+    def convert_to_object(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        real = misc.read_raw_float_data(cdata, halfsize)
+        imag = misc.read_raw_float_data(cdata2, halfsize)
+        return self.space.newcomplex(real, imag)
+
+    def convert_from_object(self, cdata, w_ob):
+        space = self.space
+        real, imag = space.unpackcomplex(w_ob)
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        misc.write_raw_float_data(cdata,  real, halfsize)
+        misc.write_raw_float_data(cdata2, imag, halfsize)
+
+    def nonzero(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        return (misc.is_nonnull_float(cdata, halfsize) |
+                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
+def eptype(name, TYPE, ctypecls, rep=1):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,6 +94,9 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
+eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
+eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
+
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,6 +8,7 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
+from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -70,6 +71,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -209,7 +212,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.
+        # trampoline functions for PyPy.  (Same with complex numbers.)
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -218,11 +221,13 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
+                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
+                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
+            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
+                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,6 +171,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -613,6 +614,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -668,6 +670,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -678,6 +682,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -731,9 +736,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -800,6 +807,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok, "_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,8 +78,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -174,37 +174,56 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
-        assert bool(cast(p, 0))
+        p = new_primitive_type(name + " _Complex")
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
-        assert cast(p, -1.1j) != cast(p, -1.1j)
+        assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1105,6 +1124,34 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
+def test_call_function_25():
+    BDouble = new_primitive_type("double")
+    BDoubleComplex = new_primitive_type("double _Complex")
+    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
@@ -3796,7 +3843,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,6 +148,8 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
+            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
+            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -273,6 +275,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,6 +1819,68 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
+    def test_function_returns_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float _Complex f1(float a, float b);",
+            "test_function_returns_float_complex", """
+            #include <complex.h>
+            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+    def test_function_returns_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double _Complex f1(double a, double b);",
+            "test_function_returns_double_complex", """
+            #include <complex.h>
+            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert result.imag == 2*5.1  # exact
+
+    def test_function_argument_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float f1(float _Complex x);",
+            "test_function_argument_float_complex", """
+            #include <complex.h>
+            static float f1(float _Complex x) { return cabsf(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-5
+        result2 = lib.f1(ffi.cast("float _Complex", x))
+        assert result2 == result
+
+    def test_function_argument_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double f1(double _Complex);",
+            "test_function_argument_double_complex", """
+            #include <complex.h>
+            static double f1(double _Complex x) { return cabs(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-11
+        result2 = lib.f1(ffi.cast("double _Complex", x))
+        assert result2 == result
+
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,6 +8,7 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -83,8 +84,9 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
-                return w_result_cdata.structobj
+                ctyperesptr = w_result_cdata.ctype
+                assert isinstance(ctyperesptr, W_CTypePointer)
+                return w_result_cdata._do_getitem(ctyperesptr, 0)
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -109,13 +111,14 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
+    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union>
+        farg = fargs[i]      # <ptr to struct/union/complex>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,15 +240,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,6 +1705,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
+            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,6 +156,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -281,6 +283,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,7 +48,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,7 +1553,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2001,6 +2002,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,15 +220,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Wed May 31 17:32:18 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 31 May 2017 14:32:18 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <592f3662.4186df0a.ec236.647a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91477:776f5fd5d727
Date: 2017-05-31 22:32 +0100
http://bitbucket.org/pypy/pypy/changeset/776f5fd5d727/

Log:	hg merge default

diff too long, truncating to 2000 out of 2270 lines

diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,7 +37,10 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            o = typ()
+            try:
+                o = typ()
+            except NotImplementedError as e:
+                self.skipTest(str(e))    # for PyPy
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/2.7/zipfile.py
@@ -622,19 +622,23 @@
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached..
         """
-        buf = ''
+        # PyPy modification: don't do repeated string concatenation
+        buf = []
+        lenbuf = 0
         if n is None:
             n = -1
         while True:
             if n < 0:
                 data = self.read1(n)
-            elif n > len(buf):
-                data = self.read1(n - len(buf))
+            elif n > lenbuf:
+                data = self.read1(n - lenbuf)
             else:
-                return buf
+                break
             if len(data) == 0:
-                return buf
-            buf += data
+                break
+            lenbuf += len(data)
+            buf.append(data)
+        return "".join(buf)
 
     def _update_crc(self, newdata, eof):
         # Update the CRC using the given data.
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -734,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1187,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1479,6 +1486,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -364,6 +364,26 @@
 
 .. __: https://bitbucket.org/pypy/pypy/issue/1974/different-behaviour-for-collections-of
 
+C-API Differences
+-----------------
+
+The external C-API has been reimplemented in PyPy as an internal cpyext module.
+We support most of the documented C-API, but sometimes internal C-abstractions
+leak out on CPython and are abused, perhaps even unknowingly. For instance,
+assignment to a ``PyTupleObject`` is not supported after the tuple is
+used internally, even by another C-API function call. On CPython this will
+succeed as long as the refcount is 1.  On PyPy this will always raise a
+``SystemError('PyTuple_SetItem called on tuple after  use of tuple")``
+exception (explicitly listed here for search engines).
+
+Another similar problem is assignment of a new function pointer to any of the
+``tp_as_*`` structures after calling ``PyType_Ready``. For instance, overriding
+``tp_as_number.nb_int`` with a different function after calling ``PyType_Ready``
+on CPython will result in the old function being called for ``x.__int__()``
+(via class ``__dict__`` lookup) and the new function being called for ``int(x)``
+(via slot lookup). On PyPy we will always call the __new__ function, not the
+old, this quirky behaviour is unfortunately necessary to fully support NumPy.
+
 Performance Differences
 -------------------------
 
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
    release-pypy2.7-v5.6.0.rst
@@ -60,6 +61,7 @@
 
 .. toctree::
 
+   release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
 
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
    whatsnew-pypy2-5.4.0.rst
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.8.0.rst
@@ -0,0 +1,155 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.8 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
+Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+v3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
+3.5.3.
+
+We continue to make incremental improvements to our C-API
+compatibility layer (cpyext). PyPy2 can now import and run many C-extension
+packages, among the most notable are Numpy, Cython, and Pandas. Performance may
+be slower than CPython, especially for frequently-called short C functions.
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work proceeds at a good pace on the PyPy3.5
+version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
+as `these benchmarks show`_ it already gives a nice speed bump.
+We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
+"PyPy3.5" supports the Python 3.5 language).
+
+CFFI_ has been updated to 1.10, improving an already great package for
+interfacing with C.
+
+As always, this release fixed many issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.8 release here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+.. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.7 released March, 2017)
+=======================================================================================
+
+See also issues that were resolved_
+
+* New features and cleanups
+
+  * Implement PyModule_New, 
+  * Fix for multiple inheritance in app-level for C-API defined classes
+  * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
+  * Document more differences with CPython here_
+  * Add native PyPy support to profile frames in vmprof
+  * Fix an issue with Exception order on failed import
+  * Fix for a corner case of __future__ imports
+
+* Bug Fixes
+
+  * Correctly handle dict.pop where the popping key is not the same type as the
+    dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
+  * Improve our file's universal newline .readline implementation for
+    ``\n``, ``\r`` confusion
+
+* Performance improvements:
+
+  * Tweaks made to improve performance by reducing the number of guards
+    inserted in jitted code, based on feedback from users
+  * Add garbage collector memory pressure to some c-level allocations
+
+* RPython improvements
+
+  * Improve the default shadowstack garbage collector, fixing a crash with
+    multithreaded code and other issues
+  * Make sure lstrip consumes the entire string
+  * Support posix_fallocate and posix_fadvise, expose them on PyPy3.5
+  * Test and fix for int_and() propagating wrong bounds
+  * Improve the generated machine code by tracking the (constant) value of
+    r11 across intructions.  This lets us avoid reloading r11 with another
+    (apparently slowish) "movabs" instruction, replacing it with either
+    nothing or a cheaper variant.
+  * Performance tweaks in the x86 JIT-generated machine code: rarely taken
+    blocks are moved off-line.  Also, the temporary register used to contain
+    large constants is reused across instructions. This helps CPUs branch
+    predictor
+
+.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+
+Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
+======================================================================
+
+* New features
+
+  * Implement main part of PEP 489 (multi-phase extension module initialization)
+  * Add docstrings to various modules and functions
+
+* Bug Fixes
+
+  * Fix inconsistencies in the xml.etree.ElementTree.Element class, which on
+    CPython is hidden by the C version from '_elementree'.
+  * OSError(None,None) is different from OSError()
+  * Get closer to supporting 32 bit windows, translation now succeeds and most
+    lib-python/3/test runs
+
+* Performance improvements:
+
+  * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+    instead of our outdated regrverbose.py script
+  * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+
+.. _resolved: whatsnew-pypy2-5.8.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,70 +1,10 @@
 ==========================
-What's new in PyPy2.7 5.8+
+What's new in PyPy2.7 5.9+
 ==========================
 
-.. this is a revision shortly after release-pypy2.7-v5.7.0
-.. startrev: 44f31f6dd39f
+.. this is a revision shortly after release-pypy2.7-v5.8.0
+.. startrev: 558bd00b3dd8
 
-Add cpyext interfaces for ``PyModule_New``
+.. branch: cffi-complex
 
-Correctly handle `dict.pop`` where the ``pop``
-key is not the same type as the ``dict``'s and ``pop``
-is called with a default (will be part of release 5.7.1)
-
-.. branch: issue2522
-
-Fix missing tp_new on w_object called through multiple inheritance
-(will be part of release 5.7.1)
-
-.. branch: lstrip_to_empty_string
-
-.. branch: vmprof-native
-
-PyPy support to profile native frames in vmprof.
-
-.. branch: reusing-r11
-.. branch: branch-prediction
-
-Performance tweaks in the x86 JIT-generated machine code: rarely taken
-blocks are moved off-line.  Also, the temporary register used to contain
-large constants is reused across instructions.
-
-.. branch: vmprof-0.4.4
-
-.. branch: controller-refactor
-
-Refactor rpython.rtyper.controllerentry.
-
-.. branch: PyBuffer-backport
-
-Internal refactoring of buffers and memoryviews. Memoryviews will now be
-accepted in a few more places, e.g. in compile().
-
-.. branch: sthalik/fix-signed-integer-sizes-1494493539409
-
-.. branch: cpyext-obj-stealing
-
-Redo much of the refcount semantics in PyList_{SG}etItem to closer match
-CPython and ensure the same PyObject stored in the list can be later
-retrieved
-
-.. branch: cpyext-recursionlimit
-
-Implement Py_EnterRecursiveCall and associated functions
-
-.. branch: pypy_ctypes_nosegfault_nofastpath
-
-Remove faulty fastpath from ctypes
-
-.. branch: sockopt_zero
-
-Passing a buffersize of 0 to socket.getsockopt
-
-.. branch: better-test-whatsnew
-
-.. branch: faster-rstruct-2
-
-Improve the performance of struct.pack and struct.pack_into by using raw_store
-or gc_store_indexed whenever possible. Moreover, enable the existing
-struct.unpack fast path to all the existing buffer types, whereas previously
-it was enabled only for strings
+Part of the upgrade to cffi 1.11
diff --git a/pypy/doc/whatsnew-pypy2-5.8.0.rst b/pypy/doc/whatsnew-pypy2-5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.8.0.rst
@@ -0,0 +1,77 @@
+==========================
+What's new in PyPy2.7 5.8+
+==========================
+
+.. this is a revision shortly after release-pypy2.7-v5.7.0
+.. startrev: 44f31f6dd39f
+
+Add cpyext interfaces for ``PyModule_New``
+
+Correctly handle `dict.pop`` where the ``pop``
+key is not the same type as the ``dict``'s and ``pop``
+is called with a default (will be part of release 5.7.1)
+
+.. branch: issue2522
+
+Fix missing tp_new on w_object called through multiple inheritance
+(will be part of release 5.7.1)
+
+.. branch: lstrip_to_empty_string
+
+.. branch: vmprof-native
+
+PyPy support to profile native frames in vmprof.
+
+.. branch: reusing-r11
+.. branch: branch-prediction
+
+Performance tweaks in the x86 JIT-generated machine code: rarely taken
+blocks are moved off-line.  Also, the temporary register used to contain
+large constants is reused across instructions.
+
+.. branch: vmprof-0.4.4
+
+.. branch: controller-refactor
+
+Refactor rpython.rtyper.controllerentry.
+
+.. branch: PyBuffer-backport
+
+Internal refactoring of buffers and memoryviews. Memoryviews will now be
+accepted in a few more places, e.g. in compile().
+
+.. branch: sthalik/fix-signed-integer-sizes-1494493539409
+
+.. branch: cpyext-obj-stealing
+
+Redo much of the refcount semantics in PyList_{SG}etItem to closer match
+CPython and ensure the same PyObject stored in the list can be later
+retrieved
+
+.. branch: cpyext-recursionlimit
+
+Implement Py_EnterRecursiveCall and associated functions
+
+.. branch: pypy_ctypes_nosegfault_nofastpath
+
+Remove faulty fastpath from ctypes
+
+.. branch: sockopt_zero
+
+Passing a buffersize of 0 to socket.getsockopt
+
+.. branch: better-test-whatsnew
+
+.. branch: faster-rstruct-2
+
+Improve the performance of struct.pack and struct.pack_into by using raw_store
+or gc_store_indexed whenever possible. Moreover, enable the existing
+struct.unpack fast path to all the existing buffer types, whereas previously
+it was enabled only for strings
+
+.. branch: Kounavi/fix-typo-depricate-to-deprecate-p-1495624547235
+
+.. branch: PyPy_profopt_enabled
+
+Add profile-based optimization option ``profopt``, and specify training data
+via ``profoptpath``
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -810,8 +810,8 @@
               '"license" for more information.', file=sys.stderr)
 
 STDLIB_WARNING = """\
-debug: WARNING: Library path not found, using compiled-in sys.path.
-debug: WARNING: 'sys.prefix' will not be set.
+debug: WARNING: Library path not found, using compiled-in sys.path, with
+debug: WARNING: sys.prefix = %r
 debug: WARNING: Make sure the pypy3 binary is kept inside its tree of files.
 debug: WARNING: It is ok to create a symlink to it from somewhere else."""
 
@@ -830,13 +830,9 @@
     executable = sys.pypy_find_executable(executable)
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
-        for lib_path in sys.path:
-            stdlib_path = sys.pypy_find_stdlib(lib_path)
-            if stdlib_path is not None:
-                break
-    if stdlib_path is None:
         initstdio()
-        print(STDLIB_WARNING, file=sys.stderr)
+        print(STDLIB_WARNING % (getattr(sys, 'prefix', '<missing>'), 
+            file=sys.stderr)
     else:
         sys.path[:] = stdlib_path
     # from this point on, we are free to use all the unicode stuff we want,
diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -641,6 +641,9 @@
         assert round(5e15) == 5e15
         assert round(-(5e15-1)) == -(5e15-1)
         assert round(-5e15) == -5e15
+        assert round(5e15/2) == 5e15/2
+        assert round((5e15+1)/2) == 5e15/2+1
+        assert round((5e15-1)/2) == 5e15/2
         #
         inf = 1e200 * 1e200
         raises(OverflowError, round, inf)
@@ -651,6 +654,12 @@
         #
         assert round(562949953421312.5, 1) == 562949953421312.5
         assert round(56294995342131.5, 3) == 56294995342131.5
+        #
+        for i in range(-10, 10):
+            expected = i if i < 0 else i + 1
+            assert round(i + 0.5) == round(i + 0.5, 0) == expected
+            x = i * 10 + 5
+            assert round(x, -1) == round(float(x), -1) == expected * 10
 
         assert round(0.0) == 0.0
         assert type(round(0.0)) == int
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.11.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,6 +91,11 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
+    def complex(self):
+        with self as ptr:
+            w_result = self.ctype.complex(ptr)
+        return w_result
+
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -405,6 +410,13 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
+    def write_raw_complex_data(self, real, imag):
+        with self as ptr:
+            halfsize = self.ctype.size >> 1
+            ptr2 = rffi.ptradd(ptr, halfsize)
+            misc.write_raw_float_data(ptr, real, halfsize)
+            misc.write_raw_float_data(ptr2, imag, halfsize)
+
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -646,6 +658,7 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
+    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,8 +105,10 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,18 +212,21 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    space = self.space
-    if self.size < 0:
-        raise oefmt(space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
+
+def _notimplemented_ffi_type(self, is_result_type, extra=''):
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s",
-                self.name, self.size, place)
+    raise oefmt(self.space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s%s",
+                self.name, self.size, place, extra)
+
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    if self.size < 0:
+        raise oefmt(self.space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
+    raise _notimplemented_ffi_type(self, is_result_type)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -260,6 +263,13 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
+def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
+    raise _notimplemented_ffi_type(self, is_result_type,
+        extra = " (the support for complex types inside libffi "
+                "is mostly missing at this point, so CFFI only "
+                "supports complex types as arguments or return "
+                "value in API-mode functions)")
+
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -276,6 +286,7 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
+W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,6 +73,14 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
+    def complex(self, cdata):
+        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+        # calling complex(), just like <cdata 'int'> cannot be directly
+        # converted by calling float()
+        space = self.space
+        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
+                    self.name)
+
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,3 +532,51 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
+
+
+class W_CTypePrimitiveComplex(W_CTypePrimitive):
+    _attrs_ = []
+
+    def cast(self, w_ob):
+        space = self.space
+        if isinstance(w_ob, cdataobj.W_CData):
+            if not isinstance(w_ob.ctype, W_CTypePrimitive):
+                raise oefmt(space.w_TypeError,
+                            "cannot cast ctype '%s' to ctype '%s'",
+                            w_ob.ctype.name, self.name)
+            w_ob = w_ob.convert_to_object()
+        #
+        imag = 0.0
+        if space.isinstance_w(w_ob, space.w_bytes):
+            real = self.cast_str(w_ob)
+        elif space.isinstance_w(w_ob, space.w_unicode):
+            real = self.cast_unicode(w_ob)
+        else:
+            real, imag = space.unpackcomplex(w_ob)
+        w_cdata = cdataobj.W_CDataMem(space, self)
+        w_cdata.write_raw_complex_data(real, imag)
+        return w_cdata
+
+    def complex(self, cdata):
+        return self.convert_to_object(cdata)
+
+    def convert_to_object(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        real = misc.read_raw_float_data(cdata, halfsize)
+        imag = misc.read_raw_float_data(cdata2, halfsize)
+        return self.space.newcomplex(real, imag)
+
+    def convert_from_object(self, cdata, w_ob):
+        space = self.space
+        real, imag = space.unpackcomplex(w_ob)
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        misc.write_raw_float_data(cdata,  real, halfsize)
+        misc.write_raw_float_data(cdata2, imag, halfsize)
+
+    def nonzero(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        return (misc.is_nonnull_float(cdata, halfsize) |
+                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
+def eptype(name, TYPE, ctypecls, rep=1):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,6 +94,9 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
+eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
+eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
+
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,6 +8,7 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
+from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -70,6 +71,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -209,7 +212,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.
+        # trampoline functions for PyPy.  (Same with complex numbers.)
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -218,11 +221,13 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
+                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
+                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
+            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
+                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,6 +171,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -613,6 +614,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -668,6 +670,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -678,6 +682,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -731,9 +736,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -800,6 +807,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok, "_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,8 +78,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -174,37 +174,56 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
-        assert bool(cast(p, 0))
+        p = new_primitive_type(name + " _Complex")
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
-        assert cast(p, -1.1j) != cast(p, -1.1j)
+        assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1105,6 +1124,34 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
+def test_call_function_25():
+    BDouble = new_primitive_type("double")
+    BDoubleComplex = new_primitive_type("double _Complex")
+    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
@@ -3796,7 +3843,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,6 +148,8 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
+            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
+            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -273,6 +275,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,6 +1819,68 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
+    def test_function_returns_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float _Complex f1(float a, float b);",
+            "test_function_returns_float_complex", """
+            #include <complex.h>
+            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+    def test_function_returns_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double _Complex f1(double a, double b);",
+            "test_function_returns_double_complex", """
+            #include <complex.h>
+            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert result.imag == 2*5.1  # exact
+
+    def test_function_argument_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float f1(float _Complex x);",
+            "test_function_argument_float_complex", """
+            #include <complex.h>
+            static float f1(float _Complex x) { return cabsf(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-5
+        result2 = lib.f1(ffi.cast("float _Complex", x))
+        assert result2 == result
+
+    def test_function_argument_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double f1(double _Complex);",
+            "test_function_argument_double_complex", """
+            #include <complex.h>
+            static double f1(double _Complex x) { return cabs(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-11
+        result2 = lib.f1(ffi.cast("double _Complex", x))
+        assert result2 == result
+
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,6 +8,7 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -83,8 +84,9 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
-                return w_result_cdata.structobj
+                ctyperesptr = w_result_cdata.ctype
+                assert isinstance(ctyperesptr, W_CTypePointer)
+                return w_result_cdata._do_getitem(ctyperesptr, 0)
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -109,13 +111,14 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
+    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union>
+        farg = fargs[i]      # <ptr to struct/union/complex>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -380,6 +380,8 @@
                 return space.newint(self.sock.getsockopt_int(level, optname))
             except SocketError as e:
                 raise converted_error(space, e)
+        if buflen < 0 or buflen > 1024:
+            raise explicit_socket_error(space, "getsockopt buflen out of range")
         return space.newbytes(self.sock.getsockopt(level, optname, buflen))
 
     def gettimeout_w(self, space):
@@ -759,6 +761,12 @@
         w_exception = space.call_function(w_exception_class, space.newtext(message))
     raise OperationError(w_exception_class, w_exception)
 
+def explicit_socket_error(space, msg):
+    w_exception_class = space.fromcache(SocketAPI).w_error
+    w_exception = space.call_function(w_exception_class, space.newtext(msg))
+    return OperationError(w_exception_class, w_exception)
+
+
 # ____________________________________________________________
 
 socketmethodnames = """
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -527,6 +527,16 @@
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
 
+    def test_getsockopt_bad_length(self):
+        import _socket
+        s = _socket.socket()
+        buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
+        assert buf == b'\x00' * 4
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
+        raises(_socket.error, s.getsockopt,
+               _socket.IPPROTO_TCP, _socket.TCP_NODELAY, -1)
+
     def test_socket_ioctl(self):
         import _socket, sys
         if sys.platform != 'win32':
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -161,3 +161,47 @@
         assert list(a) == list(range(100, 400, 100))
         assert list(a) == list(range(100, 400, 100))
         assert list(a) == list(range(100, 400, 100))
+
+    def test_setitem(self):
+        module = self.import_extension('foo', [
+            ("set_after_use", "METH_O",
+             """
+                PyObject *t2, *tuple = PyTuple_New(1);
+                PyObject * one = PyLong_FromLong(1);
+                int res;
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                Py_INCREF(args);
+                res = PyTuple_SetItem(tuple, 0, args);
+                if (res != 0)
+                {
+                    Py_DECREF(tuple);
+                    return NULL;
+                }
+                /* Do something that uses the tuple, but does not incref */
+                t2 = PyTuple_GetSlice(tuple, 0, 1);
+                Py_DECREF(t2);
+                Py_INCREF(one);
+                res = PyTuple_SetItem(tuple, 0, one);
+                Py_DECREF(tuple);
+                if (res != 0)
+                {
+                    Py_DECREF(one);
+                    return NULL;
+                }
+                Py_INCREF(Py_None);
+                return Py_None;
+             """),
+            ])
+        import sys
+        s = 'abc'
+        if '__pypy__' in sys.builtin_module_names:
+            raises(SystemError, module.set_after_use, s)
+        else:
+            module.set_after_use(s)
+
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -6,7 +6,7 @@
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
-    track_reference, make_typedescr, get_typedescr)
+    track_reference, make_typedescr, get_typedescr, pyobj_has_w_obj)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.objspace.std.tupleobject import W_TupleObject
 
@@ -132,19 +132,20 @@
 
 @cpython_api([PyObject, Py_ssize_t, PyObject], rffi.INT_real, error=-1)
 def PyTuple_SetItem(space, ref, index, py_obj):
-    # XXX this will not complain when changing tuples that have
-    # already been realized as a W_TupleObject, but won't update the
-    # W_TupleObject
     if not tuple_check_ref(space, ref):
         decref(space, py_obj)
         PyErr_BadInternalCall(space)
-    ref = rffi.cast(PyTupleObject, ref)
-    size = ref.c_ob_size
+    tupleobj = rffi.cast(PyTupleObject, ref)
+    size = tupleobj.c_ob_size
     if index < 0 or index >= size:
         decref(space, py_obj)
         raise oefmt(space.w_IndexError, "tuple assignment index out of range")
-    old_ref = ref.c_ob_item[index]
-    ref.c_ob_item[index] = py_obj    # consumes a reference
+    old_ref = tupleobj.c_ob_item[index]
+    if pyobj_has_w_obj(ref):
+        # similar but not quite equal to ref.c_ob_refcnt != 1 on CPython
+        raise oefmt(space.w_SystemError, "PyTuple_SetItem called on tuple after"
+                                        " use of tuple")
+    tupleobj.c_ob_item[index] = py_obj    # consumes a reference
     if old_ref:
         decref(space, old_ref)
     return 0
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -39,8 +39,8 @@
         'byteorder'             : 'space.newtext(sys.byteorder)',
         'maxunicode'            : 'space.newint(vm.MAXUNICODE)',
         'pypy_objspaceclass'    : 'space.newtext(repr(space))',
-        #'prefix'               : # added by pypy_initial_path() when it
-        #'exec_prefix'          : # succeeds, pointing to trunk or /usr
+        'prefix'                : 'state.get(space).w_initial_prefix',
+        'exec_prefix'           : 'state.get(space).w_initial_prefix',
         'path'                  : 'state.get(space).w_path',
         'modules'               : 'state.get(space).w_modules',
         'argv'                  : 'state.get(space).w_argv',
diff --git a/pypy/module/sys/state.py b/pypy/module/sys/state.py
--- a/pypy/module/sys/state.py
+++ b/pypy/module/sys/state.py
@@ -19,8 +19,11 @@
 
     def setinitialpath(self, space):
         from pypy.module.sys.initpath import compute_stdlib_path
+        # This initial value for sys.prefix is normally overwritten
+        # at runtime by initpath.py
+        srcdir = os.path.dirname(pypydir)
+        self.w_initial_prefix = space.newtext(srcdir)
         # Initialize the default path
-        srcdir = os.path.dirname(pypydir)
         path = compute_stdlib_path(self, srcdir)
         self.w_path = space.newlist([space.newfilename(p) for p in path])
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,15 +240,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,6 +1705,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
+            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,6 +156,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -281,6 +283,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,7 +48,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,7 +1553,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2001,6 +2002,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,15 +220,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -577,6 +577,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1) # convert -1 to -2 without creating a bridge
         return space.newint(x)
 
     def descr_eq(self, space, w_other):
diff --git a/pypy/objspace/std/test/test_bytesobject.py b/pypy/objspace/std/test/test_bytesobject.py
--- a/pypy/objspace/std/test/test_bytesobject.py
+++ b/pypy/objspace/std/test/test_bytesobject.py
@@ -149,7 +149,6 @@
             assert result == "a foo b"
             assert isinstance(result, cls)
 
-
         for format, arg, cls in [("a %s b", "foo", str),
                                  (u"a %s b", u"foo", unicode)]:
             raises(TypeError, arg.__rmod__, format[:2])
diff --git a/pypy/objspace/std/unicodeobject.py b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -323,6 +323,7 @@
 
     def descr_hash(self, space):
         x = compute_hash(self._value)
+        x -= (x == -1) # convert -1 to -2 without creating a bridge
         return space.newint(x)
 
     def descr_eq(self, space, w_other):
diff --git a/rpython/jit/backend/llsupport/test/test_gc_integration.py b/rpython/jit/backend/llsupport/test/test_gc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_gc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_gc_integration.py
@@ -463,6 +463,21 @@
     def get_root_stack_top_addr(self):
         return rffi.cast(lltype.Signed, self.stack_addr)
 
+    def getlength(self):
+        top = self.stack_addr[0]
+        base = rffi.cast(lltype.Signed, self.stack)
+        n = (top - base) // WORD
+        assert 0 <= n < 10
+        return n
+
+    def curtop(self):
+        n = self.getlength()
+        return self.stack[n - 1]
+
+    def settop(self, newvalue):
+        n = self.getlength()
+        self.stack[n - 1] = newvalue
+
 class WriteBarrierDescr(AbstractDescr):
     jit_wb_cards_set = 0
     jit_wb_if_flag_singlebyte = 1
@@ -645,7 +660,7 @@
         frames = []
 
         def check(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert len(frame.jf_frame) == self.cpu.JITFRAME_FIXED_SIZE + 4
             # we "collect"
@@ -665,14 +680,14 @@
                 assert gcmap == [22, 23, 24]
             for item, s in zip(gcmap, new_items):
                 new_frame.jf_frame[item] = rffi.cast(lltype.Signed, s)
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == rffi.cast(lltype.Signed, frame)
-            cpu.gc_ll_descr.gcrootmap.stack[0] = rffi.cast(lltype.Signed, new_frame)
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == rffi.cast(lltype.Signed, frame)
+            cpu.gc_ll_descr.gcrootmap.settop(rffi.cast(lltype.Signed, new_frame))
             print '"Collecting" moved the frame from %d to %d' % (
-                i, cpu.gc_ll_descr.gcrootmap.stack[0])
+                i, cpu.gc_ll_descr.gcrootmap.curtop())
             frames.append(new_frame)
 
         def check2(i):
-            assert cpu.gc_ll_descr.gcrootmap.stack[0] == i
+            assert cpu.gc_ll_descr.gcrootmap.curtop() == i
             frame = rffi.cast(JITFRAMEPTR, i)
             assert frame == frames[1]
             assert frame != frames[0]
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -1052,17 +1052,19 @@
 
     def _call_header_shadowstack(self, gcrootmap):
         rst = self._load_shadowstack_top_in_ebx(self.mc, gcrootmap)
-        self.mc.MOV_mr((ebx.value, 0), ebp.value)      # MOV [ebx], ebp
-        self.mc.ADD_ri(ebx.value, WORD)
+        # the '1' is to benefit from the shadowstack 'is_minor' optimization
+        self.mc.MOV_mi((ebx.value, 0), 1)               # MOV [ebx], 1
+        self.mc.MOV_mr((ebx.value, WORD), ebp.value)    # MOV [ebx + WORD], ebp
+        self.mc.ADD_ri(ebx.value, WORD * 2)
         self.mc.MOV(heap(rst), ebx)                   # MOV [rootstacktop], ebx
 
     def _call_footer_shadowstack(self, gcrootmap):
         rst = gcrootmap.get_root_stack_top_addr()
         if rx86.fits_in_32bits(rst):
-            self.mc.SUB_ji8(rst, WORD)       # SUB [rootstacktop], WORD
+            self.mc.SUB_ji8(rst, WORD * 2)       # SUB [rootstacktop], WORD * 2
         else:
             self.mc.MOV_ri(ebx.value, rst)           # MOV ebx, rootstacktop
-            self.mc.SUB_mi8((ebx.value, 0), WORD)  # SUB [ebx], WORD
+            self.mc.SUB_mi8((ebx.value, 0), WORD * 2)  # SUB [ebx], WORD * 2
 
     def redirect_call_assembler(self, oldlooptoken, newlooptoken):
         # some minimal sanity checking
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -925,6 +925,10 @@
     def gct_gc_adr_of_root_stack_top(self, hop):
         self._gc_adr_of_gcdata_attr(hop, 'root_stack_top')
 
+    def gct_gc_modified_shadowstack(self, hop):
+        # for stacklet
+        hop.genop("direct_call", [self.root_walker.gc_modified_shadowstack_ptr])
+
     def gct_gc_detach_callback_pieces(self, hop):
         op = hop.spaceop
         assert len(op.args) == 0
diff --git a/rpython/memory/gctransform/shadowstack.py b/rpython/memory/gctransform/shadowstack.py
--- a/rpython/memory/gctransform/shadowstack.py
+++ b/rpython/memory/gctransform/shadowstack.py
@@ -245,6 +245,13 @@
         from rpython.rlib import _stacklet_shadowstack

From pypy.commits at gmail.com  Wed May 31 18:18:38 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 31 May 2017 15:18:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: summarize changes since 5.7.0,
 read up to changeset 77bf5cbc29e9
Message-ID: <592f413e.10d91c0a.9c30d.5f3f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91478:72697b76c5bf
Date: 2017-06-01 01:17 +0300
http://bitbucket.org/pypy/pypy/changeset/72697b76c5bf/

Log:	summarize changes since 5.7.0, read up to changeset 77bf5cbc29e9

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -81,28 +81,52 @@
 
 See also issues that were resolved_
 
+Note that these are also merged into PyPy 3.5
+
 * New features and cleanups
 
-  * Implement PyModule_New, 
+  * Implement PyModule_New, Py_GetRecursionLimit, Py_SetRecursionLimit,
+    Py_EnterRecursiveCall, Py_LeaveRecursiveCall, populate tp_descr_get and
+    tp_descr_set slots,
+    add conversions of ``__len__``, ``__setitem__``, ``__delitem__`` to
+    appropriate C-API slots
   * Fix for multiple inheritance in app-level for C-API defined classes
   * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
   * Document more differences with CPython here_
   * Add native PyPy support to profile frames in vmprof
   * Fix an issue with Exception order on failed import
   * Fix for a corner case of __future__ imports
+  * Update packaged Windows zlib, sqlite, expat and OpenSSL to versions used
+    by CPython
+  * Allow windows builds to use ``jom.exe`` for compiling in parallel
+  * Rewrite ``itertools.groupby()``, following CPython
+  * Backport changes from PyPy 3.5 to minimize the code differences
+  * Improve support for BSD using patches contributed by downstream
+  * Support profile-guided optimization, enabled with --profopt, , and
+    specify training data ``profoptpath``
 
-* Bug Fixes
+* Bug Fixes 
 
   * Correctly handle dict.pop where the popping key is not the same type as the
     dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
   * Improve our file's universal newline .readline implementation for
     ``\n``, ``\r`` confusion
+  * Tweak issue where ctype array ``_base`` was set on empty arrays, now it
+    is closer to the implementation in CPython
+  * Fix critical bugs in shadowstack that crashed multithreaded programs and
+    very rarely showed up even in single threaded programs
+  * Remove flaky fastpath function call from ctypes
+  * Support passing a buffersize of 0 to socket.getsockopt
+  * Avoid hash() returning -1 in cpyext
 
 * Performance improvements:
 
   * Tweaks made to improve performance by reducing the number of guards
     inserted in jitted code, based on feedback from users
   * Add garbage collector memory pressure to some c-level allocations
+  * Speed up struck.pack, struck.pack_into
+  * Performance tweaks to round(x, n) for the case n == 0
+  * Improve zipfile performance by not doing repeated string concatenation
 
 * RPython improvements
 
@@ -119,6 +143,11 @@
     blocks are moved off-line.  Also, the temporary register used to contain
     large constants is reused across instructions. This helps CPUs branch
     predictor
+  * Refactor rpython.rtyper.controllerentry to use use ``@specialize`` instead
+    of ``._annspecialcase_``
+  * Refactor handling of buffers and memoryviews. Memoryviews will now be
+    accepted in a few more places, e.g. in compile()
+
 
 .. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
 
@@ -129,6 +158,15 @@
 
   * Implement main part of PEP 489 (multi-phase extension module initialization)
   * Add docstrings to various modules and functions
+  * Adapt many CPython bug/feature fixes from CPython 3.5 to PyPy3.5
+  * Translation succeeds on Mac OS X, unfortunately our buildbot slave cannot
+    be updated to the proper development versions of OpenSSL to properly
+    package a release.
+  * Implement `` _SSLSocket.server_side``
+  * Do not silently ignore ``_swappedbytes_`` in ctypes. We now raise a
+    ``NotImplementedError``
+  * Implement and expose ``msvcrt.SetErrorMode``
+  * Implement ``PyModule_GetState``
 
 * Bug Fixes
 
@@ -137,12 +175,19 @@
   * OSError(None,None) is different from OSError()
   * Get closer to supporting 32 bit windows, translation now succeeds and most
     lib-python/3/test runs
+  * Call ``sys.__interactivehook__`` at startup
 
 * Performance improvements:
 
   * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
     instead of our outdated regrverbose.py script
   * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
+  * Avoid importing the full locale module during _io initialization, 
+    CPython change fbbf8b160e8d
+  * Avoid freezing many app-level modules at translation, avoid importing many
+    modules at startup
+  * Refactor buffers, which allows an optimization for 
+    ``bytearray()[:n].tobytes()``
 
 * The following features of Python 3.5 are not implemented yet in PyPy: