From pypy.commits at gmail.com  Fri Sep  1 03:11:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 01 Sep 2017 00:11:57 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add note here about __restrict__
Message-ID: <59a9083d.01331c0a.87cae.ab36@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3007:95bd16daf438
Date: 2017-09-01 09:11 +0200
http://bitbucket.org/cffi/cffi/changeset/95bd16daf438/

Log:	Add note here about __restrict__

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -668,6 +668,10 @@
   supported at all (declare and use it as if it were an array of two
   ``long double``, and write wrapper functions in C with set_source()).
 
+* ``__restrict__`` or ``__restrict`` are extensions of, respectively,
+   GCC and MSVC.  They are not recognized.  But ``restrict`` is a C
+   keyword and is accepted (and ignored).
+
 Note that declarations like ``int field[];`` in
 structures are interpreted as variable-length structures.  Declarations
 like ``int field[...];`` on the other hand are arrays whose length is

From pypy.commits at gmail.com  Fri Sep  1 07:51:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 01 Sep 2017 04:51:06 -0700 (PDT)
Subject: [pypy-commit] pypy pycheck-macros: test,
 fix for SUBCLASS flags on types
Message-ID: <59a949aa.472e1c0a.f169f.bae4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: pycheck-macros
Changeset: r92291:a24b838be800
Date: 2017-09-01 14:49 +0300
http://bitbucket.org/pypy/pypy/changeset/a24b838be800/

Log:	test, fix for SUBCLASS flags on types

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -128,6 +128,11 @@
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()
+
+for name in ('INT', 'LONG', 'LIST', 'TUPLE', 'UNICODE', 'DICT', 'BASE_EXC',
+             'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3
+    constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name)
+
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
 globals().update(rffi_platform.configure(CConfig_constants))
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1072,7 +1072,7 @@
 
     def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
-            ("fetchFooType", "METH_VARARGS",
+            ("fetchFooType", "NOARGS",
              """
                 PyObject *o;
                 o = PyObject_New(PyObject, &Foo_Type);
@@ -1090,7 +1090,7 @@
                 Py_DECREF(e);
                 return o;
              """),
-            ("getCounter", "METH_VARARGS",
+            ("getCounter", "METH_NOARGS",
              """
                 return PyInt_FromLong(foo_counter);
              """)], prologue="""
@@ -1377,3 +1377,53 @@
         # this is equivalent to
         from collections import Hashable
         assert not isinstance(obj, Hashable)
+
+
+class AppTestFlags(AppTestCpythonExtensionBase):
+    def test_has_subclass_flag(self):
+        module = self.import_extension('foo', [
+           ("test_flags", "METH_VARARGS",
+            '''
+                long in_flag, my_flag;
+                PyObject * obj;
+                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                    return NULL;
+                if (!PyType_Check(obj))
+                {
+                    PyErr_SetString(PyExc_ValueError, "input must be type");
+                    return NULL;
+                }
+                my_flag = ((PyTypeObject*)obj)->tp_flags;
+                if ((my_flag & in_flag) != in_flag)
+                    return PyLong_FromLong(-1);
+                if (!PyType_CheckExact(obj)) {
+                    if ((my_flag & Py_TPFLAGS_TYPE_SUBCLASS) == Py_TPFLAGS_TYPE_SUBCLASS)
+                        return PyLong_FromLong(-2);
+                }
+                return PyLong_FromLong(0);
+            '''),])
+        # copied from object.h
+        Py_TPFLAGS_INT_SUBCLASS       = (1L<<23) # goes away on py3
+        Py_TPFLAGS_LONG_SUBCLASS      = (1L<<24)
+        Py_TPFLAGS_LIST_SUBCLASS      = (1L<<25)
+        Py_TPFLAGS_TUPLE_SUBCLASS     = (1L<<26)
+        Py_TPFLAGS_STRING_SUBCLASS    = (1L<<27) # rename to BYTES on py3
+        Py_TPFLAGS_UNICODE_SUBCLASS   = (1L<<28)
+        Py_TPFLAGS_DICT_SUBCLASS      = (1L<<29)
+        Py_TPFLAGS_BASE_EXC_SUBCLASS  = (1L<<30)
+        Py_TPFLAGS_TYPE_SUBCLASS      = (1L<<31)
+        for t,f in ((long, Py_TPFLAGS_LONG_SUBCLASS),
+                    (int, Py_TPFLAGS_INT_SUBCLASS),
+                    (list, Py_TPFLAGS_LIST_SUBCLASS),
+                    (tuple, Py_TPFLAGS_TUPLE_SUBCLASS),
+                    (str, Py_TPFLAGS_STRING_SUBCLASS),
+                    (unicode, Py_TPFLAGS_UNICODE_SUBCLASS),
+                    (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS),
+                    (type, Py_TPFLAGS_TYPE_SUBCLASS),
+                   ):
+            assert module.test_flags(t, f) == 0
+        class MyList(list):
+            pass
+        assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0
+
+         
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -12,12 +12,17 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
-    Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
-    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
-    Py_TPFLAGS_HAVE_INPLACEOPS, cts, parse_dir)
+    slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
+    build_type_checkers, cts, parse_dir, PyObjectFields, PyTypeObject,
+    PyTypeObjectPtr, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, Py_TPFLAGS_HAVE_INPLACEOPS,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
+    Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
+    Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
+    Py_TPFLAGS_TYPE_SUBCLASS,
+    Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
+    )
 from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -428,7 +433,7 @@
     dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space),
                                           from_ref(space, pyo), None)
 
-def inherit_special(space, pto, base_pto):
+def inherit_special(space, pto, w_obj, base_pto):
     # XXX missing: copy basicsize and flags in a magical way
     # (minimally, if tp_basicsize is zero or too low, we copy it from the base)
     if pto.c_tp_basicsize < base_pto.c_tp_basicsize:
@@ -438,6 +443,26 @@
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
 
+    #/* Setup fast subclass flags */
+    if space.issubtype_w(w_obj, space.w_Exception):
+        pto.c_tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_type):
+        pto.c_tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_int): # remove on py3
+        pto.c_tp_flags |= Py_TPFLAGS_INT_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_long):
+        pto.c_tp_flags |= Py_TPFLAGS_LONG_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_bytes): 
+        pto.c_tp_flags |= Py_TPFLAGS_STRING_SUBCLASS # STRING->BYTES on py3
+    elif space.issubtype_w(w_obj, space.w_unicode):
+        pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_tuple):
+        pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_list):
+        pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_dict):
+        pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS
+
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
         raise DescrMismatch()
@@ -939,7 +964,7 @@
     pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w))
     base = pto.c_tp_base
     if base:
-        inherit_special(space, pto, base)
+        inherit_special(space, pto, w_obj, base)
     for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)):
         if isinstance(w_base, W_TypeObject):
             inherit_slots(space, pto, w_base)

From pypy.commits at gmail.com  Fri Sep  1 09:52:27 2017
From: pypy.commits at gmail.com (afteryu)
Date: Fri, 01 Sep 2017 06:52:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: mention that deleting module attributes
 is slow (#2642)
Message-ID: <59a9661b.c88c1c0a.18948.0829@mx.google.com>

Author: afteryu <miyu.chs at gmail.com>
Branch: 
Changeset: r92292:54c318cbfcd2
Date: 2017-08-31 21:20 -0400
http://bitbucket.org/pypy/pypy/changeset/54c318cbfcd2/

Log:	mention that deleting module attributes is slow (#2642)

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -535,6 +535,11 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
+* In PyPy, module dictionaries are optimized under the assumption that deleting
+  attributes from them are rare. Because of this, e.g. ``del foo.bar`` where
+  ``foo`` is a module that contains the function ``bar``, is significantly
+  slower than CPython.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/

From pypy.commits at gmail.com  Fri Sep  1 09:52:29 2017
From: pypy.commits at gmail.com (afteryu)
Date: Fri, 01 Sep 2017 06:52:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: mention that deleting class attributes
 is slow
Message-ID: <59a9661d.50901c0a.c334f.0f2a@mx.google.com>

Author: afteryu <miyu.chs at gmail.com>
Branch: 
Changeset: r92293:6dff6e92d656
Date: 2017-09-01 08:10 -0400
http://bitbucket.org/pypy/pypy/changeset/6dff6e92d656/

Log:	mention that deleting class attributes is slow

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -535,10 +535,10 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
-* In PyPy, module dictionaries are optimized under the assumption that deleting
-  attributes from them are rare. Because of this, e.g. ``del foo.bar`` where
-  ``foo`` is a module that contains the function ``bar``, is significantly
-  slower than CPython.
+* In PyPy, module and class dictionaries are optimized under the assumption
+  that deleting attributes from them are rare. Because of this, e.g.
+  ``del foo.bar`` where ``foo`` is a module (or class) that contains the
+  function ``bar``, is significantly slower than CPython.
 
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html

From pypy.commits at gmail.com  Fri Sep  1 11:00:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Sep 2017 08:00:52 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: fix exception consistency checks
Message-ID: <59a97624.06d61c0a.110e9.2a44@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r92294:1a20a8077e2e
Date: 2017-09-01 16:00 +0100
http://bitbucket.org/pypy/pypy/changeset/1a20a8077e2e/

Log:	fix exception consistency checks

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1559,8 +1559,7 @@
         initfunc = rffi.cast(initfunctype, initptr)
         initret = generic_cpy_call_dont_convert_result(space, initfunc)
         if not initret:
-            if state.operror:
-                state.check_and_raise_exception()
+            state.check_and_raise_exception()
             raise oefmt(space.w_SystemError,
                 "initialization of %s failed without raising an exception",
                 name)
@@ -1615,7 +1614,7 @@
 @specialize.ll()
 def generic_cpy_call_dont_convert_result(space, func, *args):
     FT = lltype.typeOf(func).TO
-    return make_generic_cpy_call(FT, True, False)(space, func, *args)
+    return make_generic_cpy_call(FT, False, False)(space, func, *args)
 
 @specialize.memo()
 def make_generic_cpy_call(FT, expect_null, convert_result):
@@ -1671,10 +1670,9 @@
             cpyext_glob_tid_ptr[0] = 0
             keepalive_until_here(*keepalives)
 
-        if is_PyObject(RESULT_TYPE):
-            if not convert_result or not is_pyobj(result):
+        if convert_result and is_PyObject(RESULT_TYPE):
+            if not is_pyobj(result):
                 ret = result
-                has_result = bool(ret)
             else:
                 # The object reference returned from a C function
                 # that is called from Python must be an owned reference
@@ -1683,13 +1681,13 @@
                     ret = get_w_obj_and_decref(space, result)
                 else:
                     ret = None
-                has_result = ret is not None
 
             # Check for exception consistency
             # XXX best attempt, will miss preexisting error that is
             # overwritten with a new error of the same type
             error = PyErr_Occurred(space)
             has_new_error = (error is not None) and (error is not preexist_error)
+            has_result = ret is not None
             if not expect_null and has_new_error and has_result:
                 raise oefmt(space.w_SystemError,
                             "An exception was set, but function returned a "
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts,
-    parse_dir, bootstrap_function, generic_cpy_call, slot_function)
+    parse_dir, bootstrap_function, generic_cpy_call,
+    generic_cpy_call_dont_convert_result, slot_function)
 from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
@@ -150,21 +151,18 @@
     while cur_slot and rffi.cast(lltype.Signed, cur_slot[0].c_slot):
         if rffi.cast(lltype.Signed, cur_slot[0].c_slot) == 2:
             execf = rffi.cast(execfunctype, cur_slot[0].c_value)
-            res = generic_cpy_call(space, execf, w_mod)
-            has_error = PyErr_Occurred(space) is not None
+            res = generic_cpy_call_dont_convert_result(space, execf, w_mod)
             state = space.fromcache(State)
-            if rffi.cast(lltype.Signed, res):
-                if has_error:
-                    state.check_and_raise_exception()
-                else:
+            if res:
+                state.check_and_raise_exception()
+                raise oefmt(space.w_SystemError,
+                            "execution of module %S failed without "
+                            "setting an exception", w_mod.w_name)
+            else:
+                if state.clear_exception():
                     raise oefmt(space.w_SystemError,
-                                "execution of module %S failed without "
-                                "setting an exception", w_mod.w_name)
-            if has_error:
-                state.clear_exception()
-                raise oefmt(space.w_SystemError,
-                            "execution of module %S raised unreported "
-                            "exception", w_mod.w_name)
+                                "execution of module %S raised unreported "
+                                "exception", w_mod.w_name)
         cur_slot = rffi.ptradd(cur_slot, 1)
 
 

From pypy.commits at gmail.com  Fri Sep  1 11:36:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Sep 2017 08:36:17 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: document the branch
Message-ID: <59a97e71.46a6df0a.617e8.3261@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r92295:6c511c66aa39
Date: 2017-09-01 16:35 +0100
http://bitbucket.org/pypy/pypy/changeset/6c511c66aa39/

Log:	document the branch

diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -5,4 +5,7 @@
 .. this is the revision after release-pypy3.3-5.8.x was branched
 .. startrev: c173df164527
 
+.. branch: multiphase
 
+Implement PyType_FromSpec (PEP 384) and fix issues with PEP 489 support.
+

From pypy.commits at gmail.com  Fri Sep  1 11:39:31 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Sep 2017 08:39:31 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Close branch multiphase
Message-ID: <59a97f33.cea4df0a.26e7e.2b5a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r92296:9bfd8662eac1
Date: 2017-09-01 15:39 +0000
http://bitbucket.org/pypy/pypy/changeset/9bfd8662eac1/

Log:	Close branch multiphase


From pypy.commits at gmail.com  Fri Sep  1 11:39:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 01 Sep 2017 08:39:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Merged in multiphase (pull request #567)
Message-ID: <59a97f44.5496df0a.bab5c.317c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92297:cc9f1669c568
Date: 2017-09-01 15:39 +0000
http://bitbucket.org/pypy/pypy/changeset/cc9f1669c568/

Log:	Merged in multiphase (pull request #567)

	Multiphase

diff too long, truncating to 2000 out of 2006 lines

diff --git a/lib-python/3/test/test_importlib/extension/test_loader.py b/lib-python/3/test/test_importlib/extension/test_loader.py
--- a/lib-python/3/test/test_importlib/extension/test_loader.py
+++ b/lib-python/3/test/test_importlib/extension/test_loader.py
@@ -88,6 +88,7 @@
 
     def setUp(self):
         self.name = '_testmultiphase'
+        __import__(self.name)  # PyPy hack
         finder = self.machinery.FileFinder(None)
         self.spec = importlib.util.find_spec(self.name)
         assert self.spec
@@ -145,7 +146,8 @@
             importlib.reload(module)
             self.assertIs(ex_class, module.Example)
 
-    def test_try_registration(self):
+    # XXX: PyPy doesn't support the PyState_* functions yet
+    def XXXtest_try_registration(self):
         '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work'''
         module = self.load_module()
         with self.subTest('PyState_FindModule'):
diff --git a/lib_pypy/_testmultiphase.c b/lib_pypy/_testmultiphase.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_testmultiphase.c
@@ -0,0 +1,627 @@
+/* Copied from CPython's Modules/_testmultiphase.c */
+/***************************************************/
+
+/* Testing module for multi-phase initialization of extension modules (PEP 489)
+ */
+
+#include "Python.h"
+
+/* Example objects */
+typedef struct {
+    PyObject_HEAD
+    PyObject            *x_attr;        /* Attributes dictionary */
+} ExampleObject;
+
+/* Example methods */
+
+static int
+Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->x_attr);
+    return 0;
+}
+
+static int
+Example_finalize(ExampleObject *self)
+{
+    Py_CLEAR(self->x_attr);
+    return 0;
+}
+
+static PyObject *
+Example_demo(ExampleObject *self, PyObject *args)
+{
+    PyObject *o = NULL;
+    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+        return NULL;
+    if (o != NULL && PyUnicode_Check(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyMethodDef Example_methods[] = {
+    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+        PyDoc_STR("demo() -> None")},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static PyObject *
+Example_getattro(ExampleObject *self, PyObject *name)
+{
+    if (self->x_attr != NULL) {
+        PyObject *v = PyDict_GetItem(self->x_attr, name);
+        if (v != NULL) {
+            Py_INCREF(v);
+            return v;
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)self, name);
+}
+
+static int
+Example_setattr(ExampleObject *self, char *name, PyObject *v)
+{
+    if (self->x_attr == NULL) {
+        self->x_attr = PyDict_New();
+        if (self->x_attr == NULL)
+            return -1;
+    }
+    if (v == NULL) {
+        int rv = PyDict_DelItemString(self->x_attr, name);
+        if (rv < 0)
+            PyErr_SetString(PyExc_AttributeError,
+                "delete non-existing Example attribute");
+        return rv;
+    }
+    else
+        return PyDict_SetItemString(self->x_attr, name, v);
+}
+
+static PyType_Slot Example_Type_slots[] = {
+    {Py_tp_doc, "The Example type"},
+//    {Py_tp_finalize, Example_finalize},
+    {Py_tp_traverse, Example_traverse},
+    {Py_tp_getattro, Example_getattro},
+    {Py_tp_setattr, Example_setattr},
+    {Py_tp_methods, Example_methods},
+    {0, 0},
+};
+
+static PyType_Spec Example_Type_spec = {
+    "_testimportexec.Example",
+    sizeof(ExampleObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+    Example_Type_slots
+};
+
+/* Function of two integers returning integer */
+
+PyDoc_STRVAR(testexport_foo_doc,
+"foo(i,j)\n\
+\n\
+Return the sum of i and j.");
+
+static PyObject *
+testexport_foo(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j))
+        return NULL;
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+/* Test that PyState registration fails  */
+
+//PyDoc_STRVAR(call_state_registration_func_doc,
+//"register_state(0): call PyState_FindModule()\n\
+//register_state(1): call PyState_AddModule()\n\
+//register_state(2): call PyState_RemoveModule()");
+//
+//static PyObject *
+//call_state_registration_func(PyObject *mod, PyObject *args)
+//{
+//    int i, ret;
+//    PyModuleDef *def = PyModule_GetDef(mod);
+//    if (def == NULL) {
+//        return NULL;
+//    }
+//    if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i))
+//        return NULL;
+//    switch (i) {
+//        case 0:
+//            mod = PyState_FindModule(def);
+//            if (mod == NULL) {
+//                Py_RETURN_NONE;
+//            }
+//            return mod;
+//        case 1:
+//            ret = PyState_AddModule(mod, def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//        case 2:
+//            ret = PyState_RemoveModule(def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//    }
+//    Py_RETURN_NONE;
+//}
+
+
+static PyType_Slot Str_Type_slots[] = {
+    {Py_tp_base, NULL}, /* filled out in module exec function */
+    {0, 0},
+};
+
+static PyType_Spec Str_Type_spec = {
+    "_testimportexec.Str",
+    0,
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Str_Type_slots
+};
+
+static PyMethodDef testexport_methods[] = {
+    {"foo",             testexport_foo,         METH_VARARGS,
+        testexport_foo_doc},
+//    {"call_state_registration_func",  call_state_registration_func,
+//        METH_VARARGS, call_state_registration_func_doc},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static int execfunc(PyObject *m)
+{
+    PyObject *temp = NULL;
+
+    /* Due to cross platform compiler issues the slots must be filled
+     * here. It's required for portability to Windows without requiring
+     * C++. */
+    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+
+    /* Add a custom type */
+    temp = PyType_FromSpec(&Example_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Example", temp) != 0)
+        goto fail;
+
+    /* Add an exception type */
+    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "error", temp) != 0)
+        goto fail;
+
+    /* Add Str */
+    temp = PyType_FromSpec(&Str_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Str", temp) != 0)
+        goto fail;
+
+    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+        goto fail;
+
+    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+        goto fail;
+
+    return 0;
+ fail:
+    return -1;
+}
+
+/* Helper for module definitions; there'll be a lot of them */
+#define TEST_MODULE_DEF(name, slots, methods) { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */ \
+    name,                                       /* m_name */ \
+    PyDoc_STR("Test module " name),             /* m_doc */ \
+    0,                                          /* m_size */ \
+    methods,                                    /* m_methods */ \
+    slots,                                      /* m_slots */ \
+    NULL,                                       /* m_traverse */ \
+    NULL,                                       /* m_clear */ \
+    NULL,                                       /* m_free */ \
+}
+
+PyModuleDef_Slot main_slots[] = {
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef main_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+
+/**** Importing a non-module object ****/
+
+static PyModuleDef def_nonmodule;
+static PyModuleDef def_nonmodule_with_methods;
+
+/* Create a SimpleNamespace(three=3) */
+static PyObject*
+createfunc_nonmodule(PyObject *spec, PyModuleDef *def)
+{
+    PyObject *dct, *ns, *three;
+
+    if (def != &def_nonmodule && def != &def_nonmodule_with_methods) {
+        PyErr_SetString(PyExc_SystemError, "def does not match");
+        return NULL;
+    }
+
+    dct = PyDict_New();
+    if (dct == NULL)
+        return NULL;
+
+    three = PyLong_FromLong(3);
+    if (three == NULL) {
+        Py_DECREF(dct);
+        return NULL;
+    }
+    PyDict_SetItemString(dct, "three", three);
+    Py_DECREF(three);
+
+    ns = _PyNamespace_New(dct);
+    Py_DECREF(dct);
+    return ns;
+}
+
+static PyModuleDef_Slot slots_create_nonmodule[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule", slots_create_nonmodule, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule);
+}
+
+PyDoc_STRVAR(nonmodule_bar_doc,
+"bar(i,j)\n\
+\n\
+Return the difference of i - j.");
+
+static PyObject *
+nonmodule_bar(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:bar", &i, &j))
+        return NULL;
+    res = i - j;
+    return PyLong_FromLong(res);
+}
+
+static PyMethodDef nonmodule_methods[] = {
+    {"bar", nonmodule_bar, METH_VARARGS, nonmodule_bar_doc},
+    {NULL, NULL}           /* sentinel */
+};
+
+static PyModuleDef def_nonmodule_with_methods = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_methods", slots_create_nonmodule, nonmodule_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_methods(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_methods);
+}
+
+/**** Non-ASCII-named modules ****/
+
+static PyModuleDef def_nonascii_latin = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_latin",           /* m_name */
+    PyDoc_STR("Module named in Czech"),         /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU__testmultiphase_zkouka_naten_evc07gi8e(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_latin);
+}
+
+static PyModuleDef def_nonascii_kana = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_kana",            /* m_name */
+    PyDoc_STR("Module named in Japanese"),      /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_kana);
+}
+
+/*** Module with a single-character name ***/
+
+PyMODINIT_FUNC
+PyInit_x(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+/**** Testing NULL slots ****/
+
+static PyModuleDef null_slots_def = TEST_MODULE_DEF(
+    "_testmultiphase_null_slots", NULL, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_null_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&null_slots_def);
+}
+
+/**** Problematic modules ****/
+
+static PyModuleDef_Slot slots_bad_large[] = {
+    {_Py_mod_LAST_SLOT + 1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_large = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_large", slots_bad_large, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_large(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_large);
+}
+
+static PyModuleDef_Slot slots_bad_negative[] = {
+    {-1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_negative = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_negative", slots_bad_negative, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_negative(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_negative);
+}
+
+static PyModuleDef def_create_int_with_state = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "create_with_state",                        /* m_name */
+    PyDoc_STR("Not a PyModuleObject object, but requests per-module state"),
+    10,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_int_with_state(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_int_with_state);
+}
+
+
+static PyModuleDef def_negative_size = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "negative_size",                            /* m_name */
+    PyDoc_STR("PyModuleDef with negative m_size"),
+    -1,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_negative_size(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_negative_size);
+}
+
+
+static PyModuleDef uninitialized_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_uninitialized(PyObject *spec)
+{
+    return (PyObject*) &uninitialized_def;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_null(PyObject *spec)
+{
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_unreported_exception(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return PyModuleDef_Init(&main_def);
+}
+
+static PyObject*
+createfunc_null(PyObject *spec, PyModuleDef *def)
+{
+    return NULL;
+}
+
+PyModuleDef_Slot slots_create_null[] = {
+    {Py_mod_create, createfunc_null},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_null = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_null, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_null(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_null);
+}
+
+static PyObject*
+createfunc_raise(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return NULL;
+}
+
+static PyModuleDef_Slot slots_create_raise[] = {
+    {Py_mod_create, createfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_raise = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_raise(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_raise);
+}
+
+static PyObject*
+createfunc_unreported_exception(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return PyModule_New("foo");
+}
+
+static PyModuleDef_Slot slots_create_unreported_exception[] = {
+    {Py_mod_create, createfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_create_unreported_exception", slots_create_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_unreported_exception);
+}
+
+static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule_with_exec_slots = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_exec_slots", slots_nonmodule_with_exec_slots, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_exec_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_exec_slots);
+}
+
+static int
+execfunc_err(PyObject *mod)
+{
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_err[] = {
+    {Py_mod_exec, execfunc_err},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_err = TEST_MODULE_DEF(
+    "_testmultiphase_exec_err", slots_exec_err, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_err(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_err);
+}
+
+static int
+execfunc_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_raise[] = {
+    {Py_mod_exec, execfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_raise = TEST_MODULE_DEF(
+    "_testmultiphase_exec_raise", slots_exec_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_raise(PyObject *mod)
+{
+    return PyModuleDef_Init(&def_exec_raise);
+}
+
+static int
+execfunc_unreported_exception(PyObject *mod)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return 0;
+}
+
+static PyModuleDef_Slot slots_exec_unreported_exception[] = {
+    {Py_mod_exec, execfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_exec_unreported_exception", slots_exec_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_unreported_exception);
+}
+
+/*** Helper for imp test ***/
+
+static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit_imp_dummy(PyObject *spec)
+{
+    return PyModuleDef_Init(&imp_dummy_def);
+}
diff --git a/lib_pypy/_testmultiphase.py b/lib_pypy/_testmultiphase.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_testmultiphase.py
@@ -0,0 +1,18 @@
+import imp
+import os
+
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testmultiphase'")
+import _pypy_testcapi
+cfile = '_testmultiphase.c'
+thisdir = os.path.dirname(__file__)
+output_dir = _pypy_testcapi.get_hashed_dir(os.path.join(thisdir, cfile))
+try:
+    fp, filename, description = imp.find_module('_test_multiphase', path=[output_dir])
+    with fp:
+        imp.load_module('_testmultiphase', fp, filename, description)
+except ImportError:
+    print('could not find _testmultiphase in %s' % output_dir)
+    _pypy_testcapi.compile_shared('_testmultiphase.c', '_testmultiphase', output_dir)
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -5,4 +5,7 @@
 .. this is the revision after release-pypy3.3-5.8.x was branched
 .. startrev: c173df164527
 
+.. branch: multiphase
 
+Implement PyType_FromSpec (PEP 384) and fix issues with PEP 489 support.
+
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -26,6 +26,7 @@
 from pypy.interpreter.module import Module
 from pypy.interpreter.function import StaticMethod
 from pypy.objspace.std.sliceobject import W_SliceObject
+from pypy.objspace.std.unicodeobject import encode_object
 from pypy.module.__builtin__.descriptor import W_Property
 #from pypy.module.micronumpy.base import W_NDimArray
 from rpython.rlib.entrypoint import entrypoint_lowlevel
@@ -596,6 +597,8 @@
     'PyObject_CallFinalizerFromDealloc',
     '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack',
     'PyBytes_FromFormat', 'PyBytes_FromFormatV',
+
+    'PyType_FromSpec',
 ]
 TYPES = {}
 FORWARD_DECLS = []
@@ -1319,6 +1322,7 @@
                          source_dir / "_warnings.c",
                          source_dir / "pylifecycle.c",
                          source_dir / "object.c",
+                         source_dir / "typeobject.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
@@ -1473,12 +1477,11 @@
     # order of things here.
     from rpython.rlib import rdynload
 
-    name = space.text_w(space.getattr(w_spec, space.newtext("name")))
+    w_name = space.getattr(w_spec, space.newtext("name"))
     path = space.text_w(space.getattr(w_spec, space.newtext("origin")))
 
     if os.sep not in path:
         path = os.curdir + os.sep + path      # force a '/' in the path
-    basename = name.split('.')[-1]
     try:
         ll_libname = rffi.str2charp(path)
         try:
@@ -1486,13 +1489,14 @@
         finally:
             lltype.free(ll_libname, flavor='raw')
     except rdynload.DLOpenError as e:
-        w_name = space.newunicode(name.decode('ascii'))
         w_path = space.newfilename(path)
         raise raise_import_error(space,
             space.newfilename(e.msg), w_name, w_path)
     look_for = None
+    name = space.text_w(w_name)
     #
     if space.config.objspace.usemodules._cffi_backend:
+        basename = name.split('.')[-1]
         look_for = '_cffi_pypyinit_%s' % (basename,)
         try:
             initptr = rdynload.dlsym(dll, look_for)
@@ -1507,7 +1511,7 @@
                 raise
     #
     if space.config.objspace.usemodules.cpyext:
-        also_look_for = 'PyInit_%s' % (basename,)
+        also_look_for = get_init_name(space, w_name)
         try:
             initptr = rdynload.dlsym(dll, also_look_for)
         except KeyError:
@@ -1518,12 +1522,24 @@
             look_for += ' or ' + also_look_for
         else:
             look_for = also_look_for
+    assert look_for is not None
     msg = u"function %s not found in library %s" % (
-        unicode(look_for), space.unicode_w(space.newfilename(path)))
-    w_name = space.newunicode(name.decode('ascii'))
+        look_for.decode('utf-8'), space.unicode_w(space.newfilename(path)))
     w_path = space.newfilename(path)
     raise_import_error(space, space.newunicode(msg), w_name, w_path)
 
+def get_init_name(space, w_name):
+    name_u = space.unicode_w(w_name)
+    basename_u = name_u.split(u'.')[-1]
+    try:
+        basename = basename_u.encode('ascii')
+        return 'PyInit_%s' % (basename,)
+    except UnicodeEncodeError:
+        basename = space.bytes_w(encode_object(
+            space, space.newunicode(basename_u), 'punycode', None))
+        basename = basename.replace('-', '_')
+        return 'PyInitU_%s' % (basename,)
+
 
 initfunctype = lltype.Ptr(lltype.FuncType([], PyObject))
 
@@ -1542,7 +1558,16 @@
     try:
         initfunc = rffi.cast(initfunctype, initptr)
         initret = generic_cpy_call_dont_convert_result(space, initfunc)
-        state.check_and_raise_exception()
+        if not initret:
+            state.check_and_raise_exception()
+            raise oefmt(space.w_SystemError,
+                "initialization of %s failed without raising an exception",
+                name)
+        else:
+            if state.clear_exception():
+                raise oefmt(space.w_SystemError,
+                    "initialization of %s raised unreported exception",
+                    name)
         if not initret.c_ob_type:
             raise oefmt(space.w_SystemError,
                         "init function of %s returned uninitialized object",
@@ -1557,6 +1582,7 @@
                                                    name)
     finally:
         state.package_context = old_context
+    # XXX: should disable single-step init for non-ascii module names
     w_mod = get_w_obj_and_decref(space, initret)
     state.fixup_extension(w_mod, name, path)
     return w_mod
@@ -1570,6 +1596,9 @@
     space.getbuiltinmodule("cpyext")
     mod_as_pyobj = rawrefcount.from_obj(PyObject, w_mod)
     if mod_as_pyobj:
+        if cts.cast('PyModuleObject*', mod_as_pyobj).c_md_state:
+            # already initialised
+            return
         return exec_def(space, w_mod, mod_as_pyobj)
 
 @specialize.ll()
@@ -1641,10 +1670,9 @@
             cpyext_glob_tid_ptr[0] = 0
             keepalive_until_here(*keepalives)
 
-        if is_PyObject(RESULT_TYPE):
-            if not convert_result or not is_pyobj(result):
+        if convert_result and is_PyObject(RESULT_TYPE):
+            if not is_pyobj(result):
                 ret = result
-                has_result = bool(ret)
             else:
                 # The object reference returned from a C function
                 # that is called from Python must be an owned reference
@@ -1653,13 +1681,13 @@
                     ret = get_w_obj_and_decref(space, result)
                 else:
                     ret = None
-                has_result = ret is not None
 
             # Check for exception consistency
             # XXX best attempt, will miss preexisting error that is
             # overwritten with a new error of the same type
             error = PyErr_Occurred(space)
             has_new_error = (error is not None) and (error is not preexist_error)
+            has_result = ret is not None
             if not expect_null and has_new_error and has_result:
                 raise oefmt(space.w_SystemError,
                             "An exception was set, but function returned a "
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -84,6 +84,7 @@
 #include "pyconfig.h"
 
 #include "object.h"
+#include "typeslots.h"
 #include "abstract.h"
 #include "pymath.h"
 #include "pyport.h"
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -140,6 +140,10 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
+
+PyAPI_FUNC(PyObject*) PyType_FromSpec(PyType_Spec*);
+
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW    1       /* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -1,7 +1,8 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     cpython_api, METH_STATIC, METH_CLASS, METH_COEXIST, CANNOT_FAIL, cts,
-    parse_dir, bootstrap_function, generic_cpy_call, slot_function)
+    parse_dir, bootstrap_function, generic_cpy_call,
+    generic_cpy_call_dont_convert_result, slot_function)
 from pypy.module.cpyext.pyobject import PyObject, as_pyobj, make_typedescr
 from pypy.interpreter.module import Module
 from pypy.module.cpyext.methodobject import (
@@ -142,23 +143,26 @@
     mod = rffi.cast(PyModuleObject, mod_as_pyobj)
     moddef = mod.c_md_def
     cur_slot = rffi.cast(rffi.CArrayPtr(PyModuleDef_Slot), moddef.c_m_slots)
+    if moddef.c_m_size >= 0 and not mod.c_md_state:
+        # Always set md_state, to use as marker for exec_extension_module()
+        # (cf. CPython's PyModule_ExecDef)
+        mod.c_md_state = lltype.malloc(
+            rffi.VOIDP.TO, moddef.c_m_size, flavor='raw', zero=True)
     while cur_slot and rffi.cast(lltype.Signed, cur_slot[0].c_slot):
         if rffi.cast(lltype.Signed, cur_slot[0].c_slot) == 2:
             execf = rffi.cast(execfunctype, cur_slot[0].c_value)
-            res = generic_cpy_call(space, execf, w_mod)
-            has_error = PyErr_Occurred(space) is not None
-            if rffi.cast(lltype.Signed, res):
-                if has_error:
-                    state = space.fromcache(State)
-                    state.check_and_raise_exception()
-                else:
+            res = generic_cpy_call_dont_convert_result(space, execf, w_mod)
+            state = space.fromcache(State)
+            if res:
+                state.check_and_raise_exception()
+                raise oefmt(space.w_SystemError,
+                            "execution of module %S failed without "
+                            "setting an exception", w_mod.w_name)
+            else:
+                if state.clear_exception():
                     raise oefmt(space.w_SystemError,
-                                "execution of module %S failed without "
-                                "setting an exception", w_mod.w_name)
-            if has_error:
-                raise oefmt(space.w_SystemError,
-                            "execution of module %S raised unreported "
-                            "exception", w_mod.w_name)
+                                "execution of module %S raised unreported "
+                                "exception", w_mod.w_name)
         cur_slot = rffi.ptradd(cur_slot, 1)
 
 
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -289,11 +289,25 @@
     destructor tp_finalize;
 } PyTypeObject;
 
+typedef struct{
+    int slot;    /* slot id, see below */
+    void *pfunc; /* function pointer */
+} PyType_Slot;
+
+typedef struct{
+    const char* name;
+    int basicsize;
+    int itemsize;
+    unsigned int flags;
+    PyType_Slot *slots; /* terminated by slot==0. */
+} PyType_Spec;
+
 typedef struct _heaptypeobject {
     PyTypeObject ht_type;
+    PyAsyncMethods as_async;
     PyNumberMethods as_number;
     PyMappingMethods as_mapping;
     PySequenceMethods as_sequence;
     PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
+    PyObject *ht_name, *ht_slots, *ht_qualname;
 } PyHeapTypeObject;
diff --git a/pypy/module/cpyext/parse/typeslots.h b/pypy/module/cpyext/parse/typeslots.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/typeslots.h
@@ -0,0 +1,80 @@
+/* Do not renumber the file; these numbers are part of the stable ABI. */
+#define Py_mp_ass_subscript 3
+#define Py_mp_length 4
+#define Py_mp_subscript 5
+#define Py_nb_absolute 6
+#define Py_nb_add 7
+#define Py_nb_and 8
+#define Py_nb_bool 9
+#define Py_nb_divmod 10
+#define Py_nb_float 11
+#define Py_nb_floor_divide 12
+#define Py_nb_index 13
+#define Py_nb_inplace_add 14
+#define Py_nb_inplace_and 15
+#define Py_nb_inplace_floor_divide 16
+#define Py_nb_inplace_lshift 17
+#define Py_nb_inplace_multiply 18
+#define Py_nb_inplace_or 19
+#define Py_nb_inplace_power 20
+#define Py_nb_inplace_remainder 21
+#define Py_nb_inplace_rshift 22
+#define Py_nb_inplace_subtract 23
+#define Py_nb_inplace_true_divide 24
+#define Py_nb_inplace_xor 25
+#define Py_nb_int 26
+#define Py_nb_invert 27
+#define Py_nb_lshift 28
+#define Py_nb_multiply 29
+#define Py_nb_negative 30
+#define Py_nb_or 31
+#define Py_nb_positive 32
+#define Py_nb_power 33
+#define Py_nb_remainder 34
+#define Py_nb_rshift 35
+#define Py_nb_subtract 36
+#define Py_nb_true_divide 37
+#define Py_nb_xor 38
+#define Py_sq_ass_item 39
+#define Py_sq_concat 40
+#define Py_sq_contains 41
+#define Py_sq_inplace_concat 42
+#define Py_sq_inplace_repeat 43
+#define Py_sq_item 44
+#define Py_sq_length 45
+#define Py_sq_repeat 46
+#define Py_tp_alloc 47
+#define Py_tp_base 48
+#define Py_tp_bases 49
+#define Py_tp_call 50
+#define Py_tp_clear 51
+#define Py_tp_dealloc 52
+#define Py_tp_del 53
+#define Py_tp_descr_get 54
+#define Py_tp_descr_set 55
+#define Py_tp_doc 56
+#define Py_tp_getattr 57
+#define Py_tp_getattro 58
+#define Py_tp_hash 59
+#define Py_tp_init 60
+#define Py_tp_is_gc 61
+#define Py_tp_iter 62
+#define Py_tp_iternext 63
+#define Py_tp_methods 64
+#define Py_tp_new 65
+#define Py_tp_repr 66
+#define Py_tp_richcompare 67
+#define Py_tp_setattr 68
+#define Py_tp_setattro 69
+#define Py_tp_str 70
+#define Py_tp_traverse 71
+#define Py_tp_members 72
+#define Py_tp_getset 73
+#define Py_tp_free 74
+#define Py_nb_matrix_multiply 75
+#define Py_nb_inplace_matrix_multiply 76
+#define Py_am_await 77
+#define Py_am_aiter 78
+#define Py_am_anext 79
+/* New in 3.5 */
+#define Py_tp_finalize 80
diff --git a/pypy/module/cpyext/src/typeobject.c b/pypy/module/cpyext/src/typeobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/typeobject.c
@@ -0,0 +1,7 @@
+#include "Python.h"
+
+PyObject *
+PyType_FromSpec(PyType_Spec *spec)
+{
+    return PyType_FromSpecWithBases(spec, NULL);
+}
diff --git a/pypy/module/cpyext/test/multiphase2.c b/pypy/module/cpyext/test/multiphase2.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/multiphase2.c
@@ -0,0 +1,627 @@
+/* Copied from CPython's Modules/_testmultiphase.c */
+/***************************************************/
+
+/* Testing module for multi-phase initialization of extension modules (PEP 489)
+ */
+
+#include "Python.h"
+
+/* Example objects */
+typedef struct {
+    PyObject_HEAD
+    PyObject            *x_attr;        /* Attributes dictionary */
+} ExampleObject;
+
+/* Example methods */
+
+static int
+Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->x_attr);
+    return 0;
+}
+
+static int
+Example_finalize(ExampleObject *self)
+{
+    Py_CLEAR(self->x_attr);
+    return 0;
+}
+
+static PyObject *
+Example_demo(ExampleObject *self, PyObject *args)
+{
+    PyObject *o = NULL;
+    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+        return NULL;
+    if (o != NULL && PyUnicode_Check(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyMethodDef Example_methods[] = {
+    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+        PyDoc_STR("demo() -> None")},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static PyObject *
+Example_getattro(ExampleObject *self, PyObject *name)
+{
+    if (self->x_attr != NULL) {
+        PyObject *v = PyDict_GetItem(self->x_attr, name);
+        if (v != NULL) {
+            Py_INCREF(v);
+            return v;
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)self, name);
+}
+
+static int
+Example_setattr(ExampleObject *self, char *name, PyObject *v)
+{
+    if (self->x_attr == NULL) {
+        self->x_attr = PyDict_New();
+        if (self->x_attr == NULL)
+            return -1;
+    }
+    if (v == NULL) {
+        int rv = PyDict_DelItemString(self->x_attr, name);
+        if (rv < 0)
+            PyErr_SetString(PyExc_AttributeError,
+                "delete non-existing Example attribute");
+        return rv;
+    }
+    else
+        return PyDict_SetItemString(self->x_attr, name, v);
+}
+
+static PyType_Slot Example_Type_slots[] = {
+    {Py_tp_doc, "The Example type"},
+    {Py_tp_finalize, Example_finalize},
+    {Py_tp_traverse, Example_traverse},
+    {Py_tp_getattro, Example_getattro},
+    {Py_tp_setattr, Example_setattr},
+    {Py_tp_methods, Example_methods},
+    {0, 0},
+};
+
+static PyType_Spec Example_Type_spec = {
+    "_testimportexec.Example",
+    sizeof(ExampleObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+    Example_Type_slots
+};
+
+/* Function of two integers returning integer */
+
+PyDoc_STRVAR(testexport_foo_doc,
+"foo(i,j)\n\
+\n\
+Return the sum of i and j.");
+
+static PyObject *
+testexport_foo(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j))
+        return NULL;
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+/* Test that PyState registration fails  */
+/*
+PyDoc_STRVAR(call_state_registration_func_doc,
+"register_state(0): call PyState_FindModule()\n\
+register_state(1): call PyState_AddModule()\n\
+register_state(2): call PyState_RemoveModule()");
+
+static PyObject *
+call_state_registration_func(PyObject *mod, PyObject *args)
+{
+    int i, ret;
+    PyModuleDef *def = PyModule_GetDef(mod);
+    if (def == NULL) {
+        return NULL;
+    }
+    if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i))
+        return NULL;
+    switch (i) {
+        case 0:
+            mod = PyState_FindModule(def);
+            if (mod == NULL) {
+                Py_RETURN_NONE;
+            }
+            return mod;
+        case 1:
+            ret = PyState_AddModule(mod, def);
+            if (ret != 0) {
+                return NULL;
+            }
+            break;
+        case 2:
+            ret = PyState_RemoveModule(def);
+            if (ret != 0) {
+                return NULL;
+            }
+            break;
+    }
+    Py_RETURN_NONE;
+}
+*/
+
+static PyType_Slot Str_Type_slots[] = {
+    {Py_tp_base, NULL}, /* filled out in module exec function */
+    {0, 0},
+};
+
+static PyType_Spec Str_Type_spec = {
+    "_testimportexec.Str",
+    0,
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Str_Type_slots
+};
+
+static PyMethodDef testexport_methods[] = {
+    {"foo",             testexport_foo,         METH_VARARGS,
+        testexport_foo_doc},
+/*    {"call_state_registration_func",  call_state_registration_func,
+        METH_VARARGS, call_state_registration_func_doc},*/
+    {NULL,              NULL}           /* sentinel */
+};
+
+static int execfunc(PyObject *m)
+{
+    PyObject *temp = NULL;
+
+    /* Due to cross platform compiler issues the slots must be filled
+     * here. It's required for portability to Windows without requiring
+     * C++. */
+    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+
+    /* Add a custom type */
+    temp = PyType_FromSpec(&Example_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Example", temp) != 0)
+        goto fail;
+
+    /* Add an exception type */
+    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "error", temp) != 0)
+        goto fail;
+
+    /* Add Str */
+    temp = PyType_FromSpec(&Str_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Str", temp) != 0)
+        goto fail;
+
+    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+        goto fail;
+
+    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+        goto fail;
+
+    return 0;
+ fail:
+    return -1;
+}
+
+/* Helper for module definitions; there'll be a lot of them */
+#define TEST_MODULE_DEF(name, slots, methods) { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */ \
+    name,                                       /* m_name */ \
+    PyDoc_STR("Test module " name),             /* m_doc */ \
+    0,                                          /* m_size */ \
+    methods,                                    /* m_methods */ \
+    slots,                                      /* m_slots */ \
+    NULL,                                       /* m_traverse */ \
+    NULL,                                       /* m_clear */ \
+    NULL,                                       /* m_free */ \
+}
+
+PyModuleDef_Slot main_slots[] = {
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef main_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit_multiphase2(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+
+/**** Importing a non-module object ****/
+
+static PyModuleDef def_nonmodule;
+static PyModuleDef def_nonmodule_with_methods;
+
+/* Create a SimpleNamespace(three=3) */
+static PyObject*
+createfunc_nonmodule(PyObject *spec, PyModuleDef *def)
+{
+    PyObject *dct, *ns, *three;
+
+    if (def != &def_nonmodule && def != &def_nonmodule_with_methods) {
+        PyErr_SetString(PyExc_SystemError, "def does not match");
+        return NULL;
+    }
+
+    dct = PyDict_New();
+    if (dct == NULL)
+        return NULL;
+
+    three = PyLong_FromLong(3);
+    if (three == NULL) {
+        Py_DECREF(dct);
+        return NULL;
+    }
+    PyDict_SetItemString(dct, "three", three);
+    Py_DECREF(three);
+
+    ns = _PyNamespace_New(dct);
+    Py_DECREF(dct);
+    return ns;
+}
+
+static PyModuleDef_Slot slots_create_nonmodule[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule", slots_create_nonmodule, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule);
+}
+
+PyDoc_STRVAR(nonmodule_bar_doc,
+"bar(i,j)\n\
+\n\
+Return the difference of i - j.");
+
+static PyObject *
+nonmodule_bar(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:bar", &i, &j))
+        return NULL;
+    res = i - j;
+    return PyLong_FromLong(res);
+}
+
+static PyMethodDef nonmodule_methods[] = {
+    {"bar", nonmodule_bar, METH_VARARGS, nonmodule_bar_doc},
+    {NULL, NULL}           /* sentinel */
+};
+
+static PyModuleDef def_nonmodule_with_methods = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_methods", slots_create_nonmodule, nonmodule_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_methods(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_methods);
+}
+
+/**** Non-ASCII-named modules ****/
+
+static PyModuleDef def_nonascii_latin = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_latin",           /* m_name */
+    PyDoc_STR("Module named in Czech"),         /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU__testmultiphase_zkouka_naten_evc07gi8e(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_latin);
+}
+
+static PyModuleDef def_nonascii_kana = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_kana",            /* m_name */
+    PyDoc_STR("Module named in Japanese"),      /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_kana);
+}
+
+/*** Module with a single-character name ***/
+
+PyMODINIT_FUNC
+PyInit_x(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+/**** Testing NULL slots ****/
+
+static PyModuleDef null_slots_def = TEST_MODULE_DEF(
+    "_testmultiphase_null_slots", NULL, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_null_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&null_slots_def);
+}
+
+/**** Problematic modules ****/
+
+static PyModuleDef_Slot slots_bad_large[] = {
+    {_Py_mod_LAST_SLOT + 1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_large = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_large", slots_bad_large, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_large(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_large);
+}
+
+static PyModuleDef_Slot slots_bad_negative[] = {
+    {-1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_negative = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_negative", slots_bad_negative, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_negative(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_negative);
+}
+
+static PyModuleDef def_create_int_with_state = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "create_with_state",                        /* m_name */
+    PyDoc_STR("Not a PyModuleObject object, but requests per-module state"),
+    10,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_int_with_state(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_int_with_state);
+}
+
+
+static PyModuleDef def_negative_size = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "negative_size",                            /* m_name */
+    PyDoc_STR("PyModuleDef with negative m_size"),
+    -1,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_negative_size(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_negative_size);
+}
+
+
+static PyModuleDef uninitialized_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_uninitialized(PyObject *spec)
+{
+    return (PyObject*) &uninitialized_def;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_null(PyObject *spec)
+{
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_unreported_exception(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return PyModuleDef_Init(&main_def);
+}
+
+static PyObject*
+createfunc_null(PyObject *spec, PyModuleDef *def)
+{
+    return NULL;
+}
+
+PyModuleDef_Slot slots_create_null[] = {
+    {Py_mod_create, createfunc_null},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_null = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_null, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_null(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_null);
+}
+
+static PyObject*
+createfunc_raise(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return NULL;
+}
+
+static PyModuleDef_Slot slots_create_raise[] = {
+    {Py_mod_create, createfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_raise = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_raise(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_raise);
+}
+
+static PyObject*
+createfunc_unreported_exception(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return PyModule_New("foo");
+}
+
+static PyModuleDef_Slot slots_create_unreported_exception[] = {
+    {Py_mod_create, createfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_create_unreported_exception", slots_create_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_unreported_exception);
+}
+
+static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule_with_exec_slots = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_exec_slots", slots_nonmodule_with_exec_slots, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_exec_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_exec_slots);
+}
+
+static int
+execfunc_err(PyObject *mod)
+{
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_err[] = {
+    {Py_mod_exec, execfunc_err},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_err = TEST_MODULE_DEF(
+    "_testmultiphase_exec_err", slots_exec_err, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_err(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_err);
+}
+
+static int
+execfunc_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_raise[] = {
+    {Py_mod_exec, execfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_raise = TEST_MODULE_DEF(
+    "_testmultiphase_exec_raise", slots_exec_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_raise(PyObject *mod)
+{
+    return PyModuleDef_Init(&def_exec_raise);
+}
+
+static int
+execfunc_unreported_exception(PyObject *mod)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return 0;
+}
+
+static PyModuleDef_Slot slots_exec_unreported_exception[] = {
+    {Py_mod_exec, execfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_exec_unreported_exception", slots_exec_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_unreported_exception);
+}
+
+/*** Helper for imp test ***/
+
+static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit_imp_dummy(PyObject *spec)
+{
+    return PyModuleDef_Init(&imp_dummy_def);
+}
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -363,6 +363,8 @@
         for name in self.imported_module_names:
             self.unimport_module(name)
         self.cleanup()
+        state = self.space.fromcache(State)
+        assert not state.operror
 
 
 class AppTestCpythonExtension(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -134,3 +134,117 @@
         """
         raises(SystemError, self.import_module, name='multiphase', body=body,
                init=init)
+
+class AppTestMultiPhase2(AppTestCpythonExtensionBase):
+    def setup_class(cls):
+        cls.w_name = cls.space.wrap('multiphase2')
+        AppTestCpythonExtensionBase.setup_class.im_func(cls)
+
+    def test_multiphase2(self):
+        import sys
+        from importlib import machinery, util
+        module = self.import_module(name=self.name)
+        finder = machinery.FileFinder(None)
+        spec = util.find_spec(self.name)
+        assert spec
+        assert module.__name__ == self.name
+        #assert module.__file__ == spec.origin
+        assert module.__package__ == ''
+        raises(AttributeError, 'module.__path__')
+        assert module is sys.modules[self.name]
+        assert isinstance(module.__loader__, machinery.ExtensionFileLoader)
+
+    def test_functionality(self):
+        import types
+        module = self.import_module(name=self.name)
+        assert isinstance(module, types.ModuleType)
+        ex = module.Example()
+        assert ex.demo('abcd') == 'abcd'
+        assert ex.demo() is None
+        raises(AttributeError, 'ex.abc')
+        ex.abc = 0
+        assert ex.abc == 0
+        assert module.foo(9, 9) == 18
+        assert isinstance(module.Str(), str)
+        assert module.Str(1) + '23' == '123'
+        raises(module.error, 'raise module.error()')
+        assert module.int_const == 1969
+        assert module.str_const == 'something different'
+
+    def test_reload(self):
+        import importlib
+        module = self.import_module(name=self.name)
+        ex_class = module.Example
+        # Simulate what importlib.reload() does, without recomputing the spec
+        module.__spec__.loader.exec_module(module)
+        assert ex_class is module.Example
+
+    def w_load_from_name(self, name, origin=None, use_prefix=True):
+        from importlib import machinery, util
+        if not origin:
+            module = self.import_module(name=self.name)
+            origin = module.__loader__.path
+        if use_prefix:
+            name = '_testmultiphase_' + name
+        loader = machinery.ExtensionFileLoader(name, origin)
+        spec = util.spec_from_loader(name, loader)
+        module = util.module_from_spec(spec)
+        loader.exec_module(module)
+        return module
+
+    def test_bad_modules(self):
+        # XXX: not a very good test, since most internal issues in cpyext
+        # cause SystemErrors.
+        module = self.import_module(name=self.name)
+        origin = module.__loader__.path
+        for name in [
+                'bad_slot_large',
+                'bad_slot_negative',
+                'create_int_with_state',
+                'negative_size',
+                'create_null',
+                'create_raise',
+                'create_unreported_exception',
+                'nonmodule_with_exec_slots',
+                'exec_err',
+                'exec_raise',
+                'exec_unreported_exception',
+                ]:
+            raises(SystemError, self.load_from_name, name, origin)
+
+    def test_export_null(self):
+        excinfo = raises(SystemError, self.load_from_name, 'export_null')
+        assert "initialization" in excinfo.value.args[0]
+        assert "without raising" in excinfo.value.args[0]
+
+    def test_export_uninit(self):
+        excinfo = raises(SystemError, self.load_from_name, 'export_uninitialized')
+        assert "init function" in excinfo.value.args[0]
+        assert "uninitialized object" in excinfo.value.args[0]
+
+    def test_export_raise(self):
+        excinfo = raises(SystemError, self.load_from_name, 'export_raise')
+        assert "bad export function" == excinfo.value.args[0]
+
+    def test_export_unreported(self):
+        excinfo = raises(SystemError, self.load_from_name, 'export_unreported_exception')
+        assert "initialization" in excinfo.value.args[0]
+        assert "unreported exception" in excinfo.value.args[0]
+
+    def test_unloadable_nonascii(self):
+        name = u"fo\xf3"
+        excinfo = raises(ImportError, self.load_from_name, name)
+        assert excinfo.value.name == '_testmultiphase_' + name
+
+    def test_nonascii(self):
+        module = self.import_module(name=self.name)
+        origin = module.__loader__.path
+        cases = [
+            ('_testmultiphase_zkou\u0161ka_na\u010dten\xed', 'Czech'),
+            ('\uff3f\u30a4\u30f3\u30dd\u30fc\u30c8\u30c6\u30b9\u30c8',
+             'Japanese'),
+            ]
+        for name, lang in cases:
+            module = self.load_from_name(name, origin=origin, use_prefix=False)
+            assert module.__name__ == name
+            assert module.__doc__ == "Module named in %s" % lang
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.api import generic_cpy_call
 from pypy.module.cpyext.pyobject import make_ref, from_ref
-from pypy.module.cpyext.typeobject import PyTypeObjectPtr
+from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr
 
 import sys
 import pytest
@@ -502,6 +502,9 @@
         ref = make_ref(space, w_obj)
         api.Py_DecRef(ref)
 
+    def test_typeslots(self, space):
+        assert cts.macros['Py_tp_doc'] == 56
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def setup_class(cls):
         AppTestCpythonExtensionBase.setup_class.im_func(cls)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,5 +1,6 @@
 import os
 
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -14,17 +15,17 @@
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
     slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers,
+    build_type_checkers, Py_TPFLAGS_BASETYPE,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     cts, parse_dir)
-from pypy.module.cpyext.cparser import parse_source
+from pypy.module.cpyext.cparser import CTypeSpace
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
-    track_reference, Py_DecRef, as_pyobj)
+    track_reference, Py_DecRef, as_pyobj, incref)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
     llslot)
@@ -42,6 +43,8 @@
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
+cts.parse_header(parse_dir / "typeslots.h")
+
 
 class W_GetSetPropertyEx(GetSetProperty):
     def __init__(self, getset, w_type):
@@ -594,6 +597,7 @@
     if obj_pto.c_tp_flags & Py_TPFLAGS_HEAPTYPE:
         heaptype = rffi.cast(PyHeapTypeObject, obj)
         Py_DecRef(space, heaptype.c_ht_name)
+        Py_DecRef(space, heaptype.c_ht_qualname)
         Py_DecRef(space, base_pyo)
         _dealloc(space, obj)
 
@@ -876,6 +880,115 @@
     return generic_cpy_call(
         space, type.c_tp_alloc, type, 0)
 
+def _parse_typeslots():
+    slots_hdr = CTypeSpace()
+    slots_hdr.parse_header(parse_dir / "typeslots.h")
+    prefix2member = {
+        'tp': "ht_type",
+        'am': "as_async",
+        'nb': "as_number",
+        'mp': "as_mapping",
+        'sq': "as_sequence",
+        'bf': "as_buffer"}
+
+    TABLE = []
+    HTO = cts.gettype('PyHeapTypeObject')
+    for name, num in slots_hdr.macros.items():
+        assert isinstance(num, int)
+        assert name.startswith('Py_')
+        name = name[3:]
+        membername = 'c_' + prefix2member[name[:2]]
+        slotname = 'c_' + name
+        TARGET = HTO._flds[membername]._flds[slotname]
+        TABLE.append((num, membername, slotname, TARGET))
+    return unrolling_iterable(TABLE)
+SLOT_TABLE = _parse_typeslots()
+
+def fill_slot(ht, slotnum, ptr):
+    for num, membername, slotname, TARGET in SLOT_TABLE:
+        if num == slotnum:
+            setattr(getattr(ht, membername), slotname, rffi.cast(TARGET, ptr))
+
+
+ at cts.decl("""PyObject *
+    PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)""",
+    result_is_ll=True)
+def PyType_FromSpecWithBases(space, spec, bases):
+    from pypy.module.cpyext.unicodeobject import PyUnicode_FromString
+    res = PyType_GenericAlloc(space, space.w_type, 0)
+    res = cts.cast('PyHeapTypeObject *', res)
+    typ = res.c_ht_type
+    typ.c_tp_flags = rffi.cast(lltype.Signed, spec.c_flags)
+    typ.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    specname = rffi.charp2str(cts.cast('char*', spec.c_name))
+    dotpos = specname.rfind('.')
+    if dotpos < 0:
+        name = specname
+    else:
+        name = specname[dotpos + 1:]
+    res.c_ht_name = make_ref(space, space.newtext(name))
+    res.c_ht_qualname = res.c_ht_name
+    incref(space, res.c_ht_qualname)
+    typ.c_tp_name = spec.c_name
+    slotdefs = rffi.cast(rffi.CArrayPtr(cts.gettype('PyType_Slot')), spec.c_slots)
+    if not bases:
+        w_base = space.w_object
+        bases_w = []
+        i = 0
+        while True:
+            slotdef = slotdefs[i]
+            slotnum = rffi.cast(lltype.Signed, slotdef.c_slot)
+            if slotnum == 0:
+                break
+            elif slotnum == cts.macros['Py_tp_base']:
+                w_base = from_ref(space, cts.cast('PyObject*', slotdef.c_pfunc))
+            elif slotnum == cts.macros['Py_tp_bases']:
+                bases = cts.cast('PyObject*', slotdef.c_pfunc)
+                bases_w = space.fixedview(from_ref(space, bases))
+            i += 1
+        if not bases_w:
+            bases_w = [w_base]
+    else:
+        bases_w = space.fixedview(from_ref(space, bases))
+    w_base = best_base(space, bases_w)
+    base = cts.cast('PyTypeObject*', make_ref(space, w_base))
+    if False:  # not base.c_tp_flags & Py_TPFLAGS_BASETYPE:
+        raise oefmt(space.w_TypeError,
+            "type '%s' is not an acceptable base type",
+            rffi.charp2str(base.c_tp_name))
+
+    typ.c_tp_as_async = res.c_as_async
+    typ.c_tp_as_number = res.c_as_number
+    typ.c_tp_as_sequence = res.c_as_sequence
+    typ.c_tp_as_mapping = res.c_as_mapping
+    typ.c_tp_as_buffer = res.c_as_buffer
+    typ.c_tp_bases = bases
+    typ.c_tp_base = base
+    typ.c_tp_basicsize = cts.cast('Py_ssize_t', spec.c_basicsize)
+    typ.c_tp_itemsize = cts.cast('Py_ssize_t', spec.c_itemsize)
+
+    i = 0
+    while True:
+        slotdef = slotdefs[i]
+        slot = rffi.cast(lltype.Signed, slotdef.c_slot)
+        if slot == 0:
+            break
+        if slot < 0:  # or slot > len(slotoffsets):
+            raise oefmt(space.w_RuntimeError, "invalid slot offset")
+        if slot in (cts.macros['Py_tp_base'], cts.macros['Py_tp_bases']):
+            # Processed above
+            i += 1
+            continue
+        fill_slot(res, slot, slotdef.c_pfunc)
+        # XXX: need to make a copy of the docstring slot, which usually
+        # points to a static string literal
+        i += 1
+
+    if not typ.c_tp_dealloc:
+        typ.c_tp_dealloc = llslot(space, subtype_dealloc)
+    py_type_ready(space, typ)
+    return cts.cast('PyObject*', res)
+
 @cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)

From pypy.commits at gmail.com  Fri Sep  1 13:45:15 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:15 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: improve interaction of
 fixed_register, try_use_same_register and
Message-ID: <59a99cab.813f1c0a.2443b.14da@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92298:ffdf4c1bc430
Date: 2017-08-26 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/ffdf4c1bc430/

Log:	improve interaction of fixed_register, try_use_same_register and
	force_result_in_reg

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -63,6 +63,7 @@
 
     @specialize.arg(1)
     def foreach(self, function, arg):
+        # XXX unused?
         node = self.master_node
         while node is not None:
             function(arg, node.val)
@@ -362,6 +363,9 @@
         returns allocated register or None, if not possible.
         """
         self._check_type(v)
+        if isinstance(v, TempVar):
+            self.longevity[v] = Lifetime(self.position, self.position)
+        # YYY all subtly similar code
         assert not isinstance(v, Const)
         if selected_reg is not None:
             res = self.reg_bindings.get(v, None)
@@ -553,6 +557,7 @@
         reg = self.reg_bindings[from_v]
         del self.reg_bindings[from_v]
         self.reg_bindings[to_v] = reg
+        return reg
 
     def _move_variable_away(self, v, prev_loc):
         # YYY here we should not move it to another reg, if all uses are in
@@ -573,27 +578,35 @@
         self._check_type(result_v)
         self._check_type(v)
         if isinstance(v, Const):
-            loc = self.force_allocate_reg(result_v, forbidden_vars)
-            self.assembler.regalloc_mov(self.convert_to_imm(v), loc)
-            return loc
+            result_loc = self.force_allocate_reg(result_v, forbidden_vars)
+            self.assembler.regalloc_mov(self.convert_to_imm(v), result_loc)
+            return result_loc
         if v not in self.reg_bindings:
             # v not in a register. allocate one for result_v and move v there
-            prev_loc = self.frame_manager.loc(v)
-            loc = self.force_allocate_reg(result_v, forbidden_vars)
-            self.assembler.regalloc_mov(prev_loc, loc)
-            return loc
+            v_loc = self.frame_manager.loc(v)
+            result_loc = self.force_allocate_reg(result_v, forbidden_vars)
+            self.assembler.regalloc_mov(v_loc, result_loc)
+            return result_loc
         if self.longevity[v].last_usage > self.position:
-            # we need to find a new place for variable v and
-            # store result in the same place
-            loc = self.reg_bindings[v]
-            del self.reg_bindings[v]
-            if self.frame_manager.get(v) is None:
-                self._move_variable_away(v, loc)
-            self.reg_bindings[result_v] = loc
+            # v keeps on being live. if there is a free register, we need a
+            # move anyway, so we can use force_allocate_reg on result_v to make
+            # sure any fixed registers are used
+            if self.free_regs:
+                v_loc = self.reg_bindings[v]
+                result_loc = self.force_allocate_reg(result_v, forbidden_vars)
+                self.assembler.regalloc_mov(v_loc, result_loc)
+                return result_loc
+            else:
+                result_loc = self.reg_bindings[v]
+                if self.frame_manager.get(v) is None:
+                    v_loc = self.frame_manager.loc(v)
+                    self.assembler.regalloc_mov(result_loc, v_loc)
+                    del self.reg_bindings[v]
+                self.reg_bindings[result_v] = result_loc
+                return result_loc
         else:
-            self._reallocate_from_to(v, result_v)
-            loc = self.reg_bindings[result_v]
-        return loc
+            result_loc = self._reallocate_from_to(v, result_v)
+        return result_loc
 
     def _sync_var(self, v):
         self.assembler.num_spills += 1
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -863,13 +863,13 @@
     # use it's own class since there are so many cases
 
     def test_force_result_in_reg_1(self):
+        # var in reg, dies
         b0, b1 = newboxes(0, 0)
         longevity = {b0: Lifetime(0, 1), b1: Lifetime(1, 3)}
         fm = TFrameManager()
         asm = MockAsm()
         rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
         rm.next_instruction()
-        # first path, var is already in reg and dies
         loc0 = rm.force_allocate_reg(b0)
         rm._check_invariants()
         rm.next_instruction()
@@ -879,6 +879,7 @@
         rm._check_invariants()
 
     def test_force_result_in_reg_2(self):
+        # var in reg, survives
         b0, b1 = newboxes(0, 0)
         longevity = {b0: Lifetime(0, 2), b1: Lifetime(1, 3)}
         fm = TFrameManager()
@@ -889,12 +890,13 @@
         rm._check_invariants()
         rm.next_instruction()
         loc = rm.force_result_in_reg(b1, b0)
-        assert loc is loc0
-        assert rm.loc(b0) is not loc0
+        assert loc is not loc0
+        assert rm.loc(b0) is loc0
         assert len(asm.moves) == 1
         rm._check_invariants()
 
     def test_force_result_in_reg_3(self):
+        # var in reg, survives, no free registers
         b0, b1, b2, b3, b4 = newboxes(0, 0, 0, 0, 0)
         longevity = {b0: Lifetime(0, 2), b1: Lifetime(0, 2),
                      b3: Lifetime(0, 2), b2: Lifetime(0, 2),
@@ -929,6 +931,7 @@
         assert len(asm.moves) == 1
 
     def test_force_result_in_reg_const(self):
+        # const
         boxes, longevity = boxes_and_longevity(2)
         fm = TFrameManager()
         asm = MockAsm()
@@ -939,6 +942,50 @@
         rm.force_result_in_reg(boxes[0], c)
         rm._check_invariants()
 
+    # some tests where the result is supposed to go in a fixed register
+
+    def test_force_result_in_reg_fixed_reg_1(self):
+        # var in reg, dies
+        b0, b1 = newboxes(0, 0)
+        longevity = LifetimeManager({b0: Lifetime(0, 1), b1: Lifetime(1, 3)})
+        longevity.try_use_same_register(b0, b1)
+        longevity.fixed_register(1, r1, b1)
+        fm = TFrameManager()
+        asm = MockAsm()
+        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
+        rm.next_instruction()
+        loc0 = rm.force_allocate_reg(b0)
+        rm._check_invariants()
+        rm.next_instruction()
+        loc = rm.force_result_in_reg(b1, b0)
+        assert loc is loc0
+        assert loc is r1
+        assert len(asm.moves) == 0
+        rm._check_invariants()
+
+    def test_force_result_in_reg_fixed_reg_2(self):
+        # var in reg, survives
+        b0, b1 = newboxes(0, 0)
+        longevity = LifetimeManager({b0: Lifetime(0, 2), b1: Lifetime(1, 3)})
+
+        # has no effect, lifetimes overlap
+        longevity.try_use_same_register(b0, b1)
+        longevity.fixed_register(1, r1, b1)
+
+        fm = TFrameManager()
+        asm = MockAsm()
+        rm = RegisterManager(longevity, frame_manager=fm, assembler=asm)
+        rm.next_instruction()
+        loc0 = rm.force_allocate_reg(b0)
+        rm._check_invariants()
+        rm.next_instruction()
+        loc = rm.force_result_in_reg(b1, b0)
+        assert loc is not loc0
+        assert rm.loc(b0) is loc0
+        assert loc is r1
+        assert len(asm.moves) == 1
+        rm._check_invariants()
+
 # _____________________________________________________
 # tests that assign registers in a mocked way for a fake CPU
 
@@ -1246,6 +1293,7 @@
             ('guard_true', r8, [])
         ]
 
+    @py.test.mark.skip("messy - later")
     def test_coalescing_first_var_already_in_different_reg(self):
         ops = '''
         [i0]
@@ -1283,8 +1331,8 @@
             ('guard_false', r0, [fp0, r7, r4, r5, r6])
         ]
 
+    @py.test.mark.skip("messy - later")
     def test_call_spill(self):
-        py.test.skip("also messy")
         # i0 dies, i1 is the argument, the other fight for caller-saved regs
         # all_regs = [r0, r1, r2, r3, r4, r5, r6, r7]
         # save_around_call_regs = [r0, r1, r2, r3]
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -508,6 +508,7 @@
         # For symmetrical operations, if 'y' is already in a register
         # and won't be used after the current operation finishes,
         # then swap the role of 'x' and 'y'
+        # XXX only do that if x is *not* in a register
         if (symm and isinstance(argloc, RegLoc) and
                 self.rm.longevity[y].last_usage == self.rm.position):
             x, y = y, x
diff --git a/rpython/jit/backend/x86/vector_ext.py b/rpython/jit/backend/x86/vector_ext.py
--- a/rpython/jit/backend/x86/vector_ext.py
+++ b/rpython/jit/backend/x86/vector_ext.py
@@ -695,7 +695,7 @@
                 if not candidate_to_spill:
                     raise NoVariableToSpill
                 reg = xrm.reg_bindings[candidate_to_spill]
-                xrm._spill_var(candidate_to_spill, forbidden_vars, None)
+                xrm._spill_var(forbidden_vars, reg)
                 xrm.free_regs.append(reg)
             loc = xrm.free_regs.pop()
             self.assembler.mov(selected_reg, loc)

From pypy.commits at gmail.com  Fri Sep  1 13:45:17 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:17 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: simplification,
 remove unused function
Message-ID: <59a99cad.48ae1c0a.7da61.5c8b@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92299:786650135776
Date: 2017-08-26 09:48 +0200
http://bitbucket.org/pypy/pypy/changeset/786650135776/

Log:	simplification, remove unused function

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -559,17 +559,6 @@
         self.reg_bindings[to_v] = reg
         return reg
 
-    def _move_variable_away(self, v, prev_loc):
-        # YYY here we should not move it to another reg, if all uses are in
-        # failargs
-        if self.free_regs:
-            loc = self.free_regs.pop()
-            self.reg_bindings[v] = loc
-            self.assembler.regalloc_mov(prev_loc, loc)
-        else:
-            loc = self.frame_manager.loc(v)
-            self.assembler.regalloc_mov(prev_loc, loc)
-
     def force_result_in_reg(self, result_v, v, forbidden_vars=[]):
         """ Make sure that result is in the same register as v.
         The variable v is copied away if it's further used.  The meaning
@@ -596,17 +585,9 @@
                 result_loc = self.force_allocate_reg(result_v, forbidden_vars)
                 self.assembler.regalloc_mov(v_loc, result_loc)
                 return result_loc
-            else:
-                result_loc = self.reg_bindings[v]
-                if self.frame_manager.get(v) is None:
-                    v_loc = self.frame_manager.loc(v)
-                    self.assembler.regalloc_mov(result_loc, v_loc)
-                    del self.reg_bindings[v]
-                self.reg_bindings[result_v] = result_loc
-                return result_loc
-        else:
-            result_loc = self._reallocate_from_to(v, result_v)
-        return result_loc
+            # v needs to go to the stack. sync it there if necessary
+            self._sync_var(v)
+        return self._reallocate_from_to(v, result_v)
 
     def _sync_var(self, v):
         self.assembler.num_spills += 1

From pypy.commits at gmail.com  Fri Sep  1 13:45:20 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:20 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: fix a comment,
 something is a bit fishy about save_all_regs which is sometimes
Message-ID: <59a99cb0.a7abdf0a.33b28.6427@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92301:e0a6917be0c6
Date: 2017-09-01 19:22 +0200
http://bitbucket.org/pypy/pypy/changeset/e0a6917be0c6/

Log:	fix a comment, something is a bit fishy about save_all_regs which is
	sometimes a bool, sometimes an int

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -412,7 +412,6 @@
         v_to_spill = self._pick_variable_to_spill(forbidden_vars,
                                selected_reg, need_lower_byte=need_lower_byte)
         loc = self.reg_bindings[v_to_spill]
-        self.assembler.num_spills += 1
         self._sync_var_to_stack(v_to_spill)
         del self.reg_bindings[v_to_spill]
         return loc
@@ -623,8 +622,8 @@
         valid, but only *if they are in self.save_around_call_regs,*
         not if they are callee-saved registers!
 
-        'save_all_regs' can be 0 (default set of registers), 1 (do that
-        for all registers), or 2 (default + gc ptrs).
+        'save_all_regs' can be 0 (default set of registers), 1 (default + gc
+        ptrs), or 2 (do that for all registers).
 
         Overview of what we do (the implementation does it differently,
         for the same result):
@@ -708,6 +707,7 @@
                     break
                 assert new_reg is not None # must succeed
                 reg = self.reg_bindings[v]
+                self.assembler.num_moves_calls += 1
                 self.assembler.regalloc_mov(reg, new_reg)
                 self.reg_bindings[v] = new_reg    # change the binding
                 new_free_regs.append(reg)
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -816,6 +816,7 @@
             # without distinguishing call sites, which we don't do any
             # more for now.
             if gcrootmap: # and gcrootmap.is_shadow_stack:
+                # YYY this is weird???
                 save_all_regs = 2
         self.rm.before_call(save_all_regs=save_all_regs)
         if op.type != 'v':

From pypy.commits at gmail.com  Fri Sep  1 13:45:24 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:24 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: in-progress and very
 hackish atm: add register hints in the x86 backend
Message-ID: <59a99cb4.ceb1df0a.14c7e.46ae@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92303:4cd14465e80c
Date: 2017-09-01 19:44 +0200
http://bitbucket.org/pypy/pypy/changeset/4cd14465e80c/

Log:	in-progress and very hackish atm: add register hints in the x86
	backend

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -158,6 +158,7 @@
         self.final_jump_op = None
 
     def _prepare(self, inputargs, operations, allgcrefs):
+        from rpython.jit.backend.x86.reghint import X86RegisterHints
         for box in inputargs:
             assert box.get_forwarded() is None
         cpu = self.assembler.cpu
@@ -166,6 +167,7 @@
                                                        allgcrefs)
         # compute longevity of variables
         longevity = compute_vars_longevity(inputargs, operations)
+        X86RegisterHints().add_hints(longevity, inputargs, operations)
         self.longevity = longevity
         self.rm = gpr_reg_mgr_cls(self.longevity,
                                   frame_manager = self.fm,
@@ -966,6 +968,8 @@
             # Load the register for the result.  Possibly reuse 'args[0]'.
             # But the old value of args[0], if it survives, is first
             # spilled away.  We can't overwrite any of op.args[2:] here.
+
+            # YYY args[0] is maybe not spilled here!!!
             resloc = self.rm.force_result_in_reg(op, args[0],
                                                  forbidden_vars=args[2:])
 
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/reghint.py
@@ -0,0 +1,195 @@
+from rpython.jit.codewriter.effectinfo import EffectInfo
+from rpython.jit.metainterp.resoperation import rop
+from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr,
+    ConstFloat, INT, REF, FLOAT, VECTOR, TargetToken, AbstractFailDescr)
+from rpython.jit.backend.llsupport.descr import CallDescr
+from rpython.jit.backend.x86.regloc import (FrameLoc, RegLoc, ConstFloatLoc,
+    FloatImmedLoc, ImmedLoc, imm, imm0, imm1, ecx, eax, edx, ebx, esi, edi,
+    ebp, r8, r9, r10, r11, r12, r13, r14, r15, xmm0, xmm1, xmm2, xmm3, xmm4,
+    xmm5, xmm6, xmm7, xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14,
+    X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,)
+from rpython.jit.backend.x86 import rx86
+
+
+from rpython.jit.backend.x86.regalloc import (
+    X86_64_RegisterManager, X86_64_XMMRegisterManager)
+
+# tell the register allocator hints about which variables should be placed in
+# what registers (those are just hints, the register allocator will try its
+# best to achieve that).
+
+
+class X86RegisterHints(object):
+    def add_hints(self, longevity, inputargs, operations):
+        self.longevity = longevity
+        for i in range(len(operations)):
+            op = operations[i]
+            oplist[op.getopnum()](self, op, i)
+
+    def not_implemented_op(self, op, position):
+        # no hint by default
+        pass
+
+    def _consider_binop_part(self, op, position, symm=False):
+        x = op.getarg(0)
+        y = op.getarg(1)
+
+        # For symmetrical operations, if 'y' is already in a register
+        # and won't be used after the current operation finishes,
+        # then swap the role of 'x' and 'y'
+        if self.longevity[x].last_usage > position:
+            if self.longevity[y].last_usage == position:
+                x, y = y, x
+                self.longevity.try_use_same_register(y, op)
+        else:
+            self.longevity.try_use_same_register(x, op)
+
+    def _consider_binop(self, op, position):
+        self._consider_binop_part(op, position)
+
+    def _consider_binop_symm(self, op, position):
+        self._consider_binop_part(op, position, symm=True)
+
+    #consider_int_mul = _consider_binop_symm
+    #consider_int_and = _consider_binop_symm
+    #consider_int_or  = _consider_binop_symm
+    #consider_int_xor = _consider_binop_symm
+
+    #consider_int_mul_ovf = _consider_binop_symm
+    #consider_int_sub_ovf = _consider_binop
+    consider_int_add_ovf = _consider_binop_symm
+
+    def _consider_lea(self, op, loc):
+        argloc = self.loc(op.getarg(1))
+        resloc = self.force_allocate_reg(op)
+        self.perform(op, [loc, argloc], resloc)
+
+    def consider_int_add(self, op, position):
+        y = op.getarg(1)
+        if isinstance(y, ConstInt) and rx86.fits_in_32bits(y.value):
+            pass # nothing to be hinted
+        else:
+            self._consider_binop_symm(op, position)
+
+    consider_nursery_ptr_increment = consider_int_add
+
+    def Xconsider_int_lshift(self, op, position):
+        if not isinstance(op.getarg(1), Const):
+            self.longevity.fixed_register(position, ecx, op.getarg(1))
+
+    #consider_int_rshift  = consider_int_lshift
+    #consider_uint_rshift = consider_int_lshift
+
+    def Xconsider_call_malloc_nursery(self, op, position):
+        self.longevity.fixed_register(position, ecx, op)
+        self.longevity.fixed_register(position, edx)
+
+    #consider_call_malloc_nursery_varsize = consider_call_malloc_nursery
+    #consider_call_malloc_nursery_varsize_frame = consider_call_malloc_nursery
+
+
+    def _call(self, op, position, args, save_all_regs=False):
+        # XXX fish for correct argtypes
+        CallHints64(self.longevity).hint(position, args, [], save_all_regs)
+
+    def _consider_call(self, op, position, guard_not_forced=False, first_arg_index=1):
+        calldescr = op.getdescr()
+        assert isinstance(calldescr, CallDescr)
+        effectinfo = calldescr.get_extra_info()
+        # XXX this is nonsense, share the code somehow
+        if guard_not_forced:
+            gc_level = 2
+        elif effectinfo is None or effectinfo.check_can_collect():
+            gc_level = 1
+        else:
+            gc_level = 0
+        self._call(op, position, op.getarglist()[first_arg_index:],
+                   save_all_regs=gc_level)
+
+    def _consider_real_call(self, op, position):
+        effectinfo = op.getdescr().get_extra_info()
+        assert effectinfo is not None
+        oopspecindex = effectinfo.oopspecindex
+        if oopspecindex != EffectInfo.OS_NONE:
+            raise NotImplementedError
+        self._consider_call(op, position)
+
+    consider_call_i = _consider_real_call
+    consider_call_r = _consider_real_call
+    consider_call_f = _consider_real_call
+    consider_call_n = _consider_real_call
+
+oplist = [X86RegisterHints.not_implemented_op] * rop._LAST
+
+for name, value in X86RegisterHints.__dict__.iteritems():
+    if name.startswith('consider_'):
+        name = name[len('consider_'):]
+        num = getattr(rop, name.upper())
+        oplist[num] = value
+
+
+
+class CallHints64(object):
+
+    ARGUMENTS_GPR = [edi, esi, edx, ecx, r8, r9]
+    ARGUMENTS_XMM = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7]
+    _ALL_CALLEE_SAVE_GPR = [ebx, r12, r13, r14, r15]
+
+    next_arg_gpr = 0
+    next_arg_xmm = 0
+
+    def __init__(self, longevity):
+        self.longevity = longevity
+
+    def _unused_gpr(self):
+        i = self.next_arg_gpr
+        self.next_arg_gpr = i + 1
+        try:
+            res = self.ARGUMENTS_GPR[i]
+        except IndexError:
+            return None
+        return res
+
+    def _unused_xmm(self):
+        i = self.next_arg_xmm
+        self.next_arg_xmm = i + 1
+        try:
+            return self.ARGUMENTS_XMM[i]
+        except IndexError:
+            return None
+
+    def hint(self, position, args, argtypes, save_all_regs):
+        import pdb; pdb.set_trace()
+        hinted_xmm = []
+        hinted_gpr = []
+        for i in range(len(args)):
+            arg = args[i]
+            if arg.type == "f":
+                tgt = self._unused_xmm()
+                if tgt is not None and not arg.is_constant():
+                    self.longevity.fixed_register(position, tgt, arg)
+                    hinted_xmm.append(tgt)
+            elif i < len(argtypes) and argtypes[i] == 'S':
+                # Singlefloat argument
+                tgt = self._unused_xmm()
+                if tgt is not None and not arg.is_constant():
+                    self.longevity.fixed_register(position, tgt, arg)
+                    hinted_xmm.append(tgt)
+            else:
+                tgt = self._unused_gpr()
+                if tgt is not None and not arg.is_constant():
+                    self.longevity.fixed_register(position, tgt, arg)
+                    hinted_gpr.append(tgt)
+        # block all remaining registers that are not caller save
+        # XXX the case save_all_regs == 1 (save callee-save regs + gc ptrs) is
+        # no expressible atm
+        if save_all_regs == 2:
+            regs = X86_64_RegisterManager.all_regs
+        else:
+            regs = X86_64_RegisterManager.save_around_call_regs
+        for reg in regs:
+            if reg not in hinted_gpr:
+                self.longevity.fixed_register(position, reg)
+        for reg in X86_64_XMMRegisterManager.all_regs:
+            if reg not in hinted_xmm:
+                self.longevity.fixed_register(position, reg)
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
new file mode 100644
--- /dev/null
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -0,0 +1,72 @@
+""" annoying integration tests for register allocation in the x86 backend """
+
+from rpython.jit.backend.llsupport.test import test_regalloc_integration
+from rpython.jit.backend.x86.assembler import Assembler386
+
+class LogEntry(object):
+    def __init__(self, position, name, *args):
+        self.position = position
+        self.name = name
+        self.args = args
+
+    def __repr__(self):
+        r = repr(self.args)
+        if self.name == "op":
+            r = repr(self.args[1:])
+            return "<%s: %s %s %s>" % (self.position, self.name, self.args[0], r.strip("(),"))
+        return "<%s: %s %s>" % (self.position, self.name, r.strip("(),"))
+
+class LoggingAssembler(Assembler386):
+    def __init__(self, *args, **kwargs):
+        self._instr_log = []
+        Assembler386.__init__(self, *args, **kwargs)
+
+    def _log(self, name, *args):
+        self._instr_log.append(LogEntry(self._regalloc.rm.position, name, *args))
+
+    def mov(self, from_loc, to_loc):
+        self._log("mov", from_loc, to_loc)
+        return Assembler386.mov(self, from_loc, to_loc)
+
+    def regalloc_mov(self, from_loc, to_loc):
+        import pdb; pdb.set_trace()
+        self._log("mov", from_loc, to_loc)
+        return Assembler386.mov(self, from_loc, to_loc)
+
+    def regalloc_perform(self, op, arglocs, resloc):
+        self._log("op", op.getopname(), arglocs, resloc)
+        return Assembler386.regalloc_perform(self, op, arglocs, resloc)
+
+    def regalloc_perform_discard(self, op, arglocs):
+        self._log("op", op.getopname(), arglocs)
+        return Assembler386.regalloc_perform_discard(self, op, arglocs)
+
+    def regalloc_perform_guard(self, guard_op, faillocs, arglocs, resloc,
+                               frame_depth):
+        self._log("guard", guard_op.getopname(), arglocs, faillocs, resloc)
+        return Assembler386.regalloc_perform_guard(self, guard_op, faillocs,
+                arglocs, resloc, frame_depth)
+
+
+class TestCheckRegistersExplicitly(test_regalloc_integration.BaseTestRegalloc):
+    def setup_class(cls):
+        cls.cpu.assembler = LoggingAssembler(cls.cpu, False)
+        cls.cpu.assembler.setup_once()
+
+    def setup_method(self, meth):
+        self.cpu.assembler._instr_log = self.log = []
+
+    def test_call_use_correct_regs(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1)
+        i8 = int_add(i2, 13)
+        i9 = call_i(ConstClass(f1ptr), i7, descr=f1_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) [i8]
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        # two moves are needed from the stack frame to registers for arguments
+        # i0 and i1 one for the result to the stack
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 3

From pypy.commits at gmail.com  Fri Sep  1 13:45:19 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:19 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: even shorter. also rename
 _sync_var to _sync_var_to_stack to make clear what it does
Message-ID: <59a99caf.48badf0a.390f2.575c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92300:a855c810d86c
Date: 2017-08-26 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/a855c810d86c/

Log:	even shorter. also rename _sync_var to _sync_var_to_stack to make
	clear what it does

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -413,7 +413,7 @@
                                selected_reg, need_lower_byte=need_lower_byte)
         loc = self.reg_bindings[v_to_spill]
         self.assembler.num_spills += 1
-        self._sync_var(v_to_spill)
+        self._sync_var_to_stack(v_to_spill)
         del self.reg_bindings[v_to_spill]
         return loc
 
@@ -494,7 +494,7 @@
         self.bindings_to_frame_reg[v] = None
 
     def force_spill_var(self, var):
-        self._sync_var(var)
+        self._sync_var_to_stack(var)
         try:
             loc = self.reg_bindings[var]
             del self.reg_bindings[var]
@@ -570,26 +570,25 @@
             result_loc = self.force_allocate_reg(result_v, forbidden_vars)
             self.assembler.regalloc_mov(self.convert_to_imm(v), result_loc)
             return result_loc
-        if v not in self.reg_bindings:
-            # v not in a register. allocate one for result_v and move v there
-            v_loc = self.frame_manager.loc(v)
+        v_keeps_living = self.longevity[v].last_usage > self.position
+        # there are two cases where we should allocate a new register for
+        # result:
+        # 1) v is itself not in a register
+        # 2) v keeps on being live. if there is a free register, we need a move
+        # anyway, so we can use force_allocate_reg on result_v to make sure any
+        # fixed registers are used
+        if (v not in self.reg_bindings or (v_keeps_living and self.free_regs)):
+            v_loc = self.loc(v)
             result_loc = self.force_allocate_reg(result_v, forbidden_vars)
             self.assembler.regalloc_mov(v_loc, result_loc)
             return result_loc
-        if self.longevity[v].last_usage > self.position:
-            # v keeps on being live. if there is a free register, we need a
-            # move anyway, so we can use force_allocate_reg on result_v to make
-            # sure any fixed registers are used
-            if self.free_regs:
-                v_loc = self.reg_bindings[v]
-                result_loc = self.force_allocate_reg(result_v, forbidden_vars)
-                self.assembler.regalloc_mov(v_loc, result_loc)
-                return result_loc
-            # v needs to go to the stack. sync it there if necessary
-            self._sync_var(v)
+        if v_keeps_living:
+            # since there are no free registers, v needs to go to the stack.
+            # sync it there.
+            self._sync_var_to_stack(v)
         return self._reallocate_from_to(v, result_v)
 
-    def _sync_var(self, v):
+    def _sync_var_to_stack(self, v):
         self.assembler.num_spills += 1
         if not self.frame_manager.get(v):
             reg = self.reg_bindings[v]
@@ -600,7 +599,7 @@
         # otherwise it's clean
 
     def _bc_spill(self, v, new_free_regs):
-        self._sync_var(v)
+        self._sync_var_to_stack(v)
         new_free_regs.append(self.reg_bindings.pop(v))
 
     def before_call(self, force_store=[], save_all_regs=0):
@@ -788,7 +787,7 @@
         descr = op.getdescr()
         assert isinstance(descr, JitCellToken)
         if op.numargs() == 2:
-            self.rm._sync_var(op.getarg(1))
+            self.rm._sync_var_to_stack(op.getarg(1))
             return [self.loc(op.getarg(0)), self.fm.loc(op.getarg(1))]
         else:
             assert op.numargs() == 1

From pypy.commits at gmail.com  Fri Sep  1 13:45:22 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 01 Sep 2017 10:45:22 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: move two tests to make them
 run by the x86 test file too
Message-ID: <59a99cb2.8683df0a.746e2.4cd3@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92302:3162799170f9
Date: 2017-09-01 19:42 +0200
http://bitbucket.org/pypy/pypy/changeset/3162799170f9/

Log:	move two tests to make them run by the x86 test file too

diff --git a/rpython/jit/metainterp/test/test_list.py b/rpython/jit/metainterp/test/test_list.py
--- a/rpython/jit/metainterp/test/test_list.py
+++ b/rpython/jit/metainterp/test/test_list.py
@@ -336,6 +336,35 @@
                            'guard_true': 2,
                            'jump': 1})
 
+    def test_conditional_call_append(self):
+        jitdriver = JitDriver(greens = [], reds = 'auto')
+
+        def f(n):
+            l = []
+            while n > 0:
+                jitdriver.jit_merge_point()
+                l.append(n)
+                n -= 1
+            return len(l)
+
+        res = self.meta_interp(f, [10])
+        assert res == 10
+        self.check_resops(call=0, cond_call=2)
+
+    def test_conditional_call_pop(self):
+        jitdriver = JitDriver(greens = [], reds = 'auto')
+
+        def f(n):
+            l = range(n)
+            while n > 0:
+                jitdriver.jit_merge_point()
+                l.pop()
+                n -= 1
+            return len(l)
+
+        res = self.meta_interp(f, [10])
+        assert res == 0
+        self.check_resops(call=0, cond_call=2)
 
 class TestLLtype(ListTests, LLJitMixin):
     def test_listops_dont_invalidate_caches(self):
@@ -366,36 +395,6 @@
         # itself
         self.check_resops(getfield_gc_i=2, getfield_gc_r=5)
 
-    def test_conditional_call_append(self):
-        jitdriver = JitDriver(greens = [], reds = 'auto')
-
-        def f(n):
-            l = []
-            while n > 0:
-                jitdriver.jit_merge_point()
-                l.append(n)
-                n -= 1
-            return len(l)
-
-        res = self.meta_interp(f, [10])
-        assert res == 10
-        self.check_resops(call=0, cond_call=2)
-
-    def test_conditional_call_pop(self):
-        jitdriver = JitDriver(greens = [], reds = 'auto')
-
-        def f(n):
-            l = range(n)
-            while n > 0:
-                jitdriver.jit_merge_point()
-                l.pop()
-                n -= 1
-            return len(l)
-
-        res = self.meta_interp(f, [10])
-        assert res == 0
-        self.check_resops(call=0, cond_call=2)
-
     def test_zero_init_resizable(self):
         def f(n):
             l = [0] * n

From pypy.commits at gmail.com  Sat Sep  2 05:24:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Sep 2017 02:24:13 -0700 (PDT)
Subject: [pypy-commit] pypy default: PR #565, originally by splasky
Message-ID: <59aa78bd.02da1c0a.1344c.db33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92304:e9c0dff56c4c
Date: 2017-09-02 11:23 +0200
http://bitbucket.org/pypy/pypy/changeset/e9c0dff56c4c/

Log:	PR #565, originally by splasky

	Add Ubuntu 17.04 (and likely Debian) build time dependencies

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,8 +119,15 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian and Ubuntu, this is the command to install all build-time
-dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \

From pypy.commits at gmail.com  Sat Sep  2 05:34:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 02 Sep 2017 02:34:06 -0700 (PDT)
Subject: [pypy-commit] pypy nogil-unsafe-2: Rename the operation to reflect
 the readonly-ness
Message-ID: <59aa7b0e.84a3df0a.22016.2547@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: nogil-unsafe-2
Changeset: r92305:4698de95c523
Date: 2017-09-02 11:33 +0200
http://bitbucket.org/pypy/pypy/changeset/4698de95c523/

Log:	Rename the operation to reflect the readonly-ness

diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -433,7 +433,7 @@
 
         def _trace_tlref(gc, obj, callback, arg):
             p = llmemory.NULL
-            llop.threadlocalref_acquire(lltype.Void)
+            llop.threadlocalref_readonly_acquire(lltype.Void)
             while True:
                 p = llop.threadlocalref_enum(llmemory.Address, p)
                 if not p:
@@ -490,7 +490,7 @@
 @specialize.arg(0)
 def enum_all_threadlocals(callback, arg):
     p = llmemory.NULL
-    llop.threadlocalref_acquire(lltype.Void)
+    llop.threadlocalref_readonly_acquire(lltype.Void)
     while True:
         p = llop.threadlocalref_enum(llmemory.Address, p)
         if not p:
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -997,7 +997,7 @@
         return self.op_raw_load(RESTYPE, _address_of_thread_local(), offset)
     op_threadlocalref_get.need_result_type = True
 
-    def op_threadlocalref_acquire(self, prev):
+    def op_threadlocalref_readonly_acquire(self, prev):
         raise NotImplementedError
     def op_threadlocalref_release(self, prev):
         raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -551,8 +551,8 @@
 
     'threadlocalref_addr':  LLOp(),                   # get (or make) addr of tl
     'threadlocalref_get':   LLOp(sideeffects=False),  # read field (no check)
-    'threadlocalref_acquire':  LLOp(),                # lock for enum
-    'threadlocalref_release':  LLOp(),                # lock for enum
+    'threadlocalref_readonly_acquire':  LLOp(),       # lock for enum
+    'threadlocalref_release':           LLOp(),       # lock for enum
     'threadlocalref_enum':  LLOp(sideeffects=False),  # enum all threadlocalrefs
 
     # __________ debugging __________
diff --git a/rpython/translator/c/src/thread_gil.c b/rpython/translator/c/src/thread_gil.c
--- a/rpython/translator/c/src/thread_gil.c
+++ b/rpython/translator/c/src/thread_gil.c
@@ -111,7 +111,7 @@
     assert(counter_of_sevens == 0);
 
     /* signal all threads to enter safepoints */
-    OP_THREADLOCALREF_ACQUIRE(/* */);
+    OP_THREADLOCALREF_READONLY_ACQUIRE(/* */);
 
     struct pypy_threadlocal_s *t = NULL;
     while (1) {
diff --git a/rpython/translator/c/src/threadlocal.h b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -31,7 +31,8 @@
 /* will return the head of the list */
 RPY_EXTERN struct pypy_threadlocal_s *_RPython_ThreadLocals_Head();
 
-#define OP_THREADLOCALREF_ACQUIRE(r)   _RPython_ThreadLocals_ReadOnlyAcquire()
+#define OP_THREADLOCALREF_READONLY_ACQUIRE(r)     \
+                                       _RPython_ThreadLocals_ReadOnlyAcquire()
 #define OP_THREADLOCALREF_RELEASE(r)   _RPython_ThreadLocals_Release()
 #define OP_THREADLOCALREF_ENUM(p, r)   r = _RPython_ThreadLocals_Enum(p)
 

From pypy.commits at gmail.com  Sun Sep  3 03:12:43 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Sep 2017 00:12:43 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: fix an off-by-one error in
 register selection
Message-ID: <59abab6b.06991c0a.10ad5.6fba@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92306:69fc3fac05ad
Date: 2017-09-03 08:47 +0200
http://bitbucket.org/pypy/pypy/changeset/69fc3fac05ad/

Log:	fix an off-by-one error in register selection

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -985,7 +985,7 @@
                 unfixed_reg = reg
                 continue
             use_after = fixed_reg_pos.free_until_pos(position)
-            if use_after < longevityvar.last_usage_including_sharing():
+            if use_after <= longevityvar.last_usage_including_sharing():
                 # can't fit
                 continue
             assert use_after >= longevityvar.last_usage_including_sharing()
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -308,6 +308,17 @@
     loc = longevity.try_pick_free_reg(0, b1, [r0, r1])
     assert loc == r1
 
+def test_try_pick_free_reg_bug2():
+    b0, b1, b2, b3, b4 = newboxes(0, 0, 0, 0, 0)
+    l0 = Lifetime(1, 2)
+    l1 = Lifetime(2, 4)
+    longevity = LifetimeManager({b0: l0, b1: l1})
+    longevity.fixed_register(4, r1, b1)
+
+    # does not fit into r0, use r1
+    loc = longevity.try_pick_free_reg(0, b0, [r0, r1])
+    assert loc == r0
+
 def test_simple_coalescing():
     b0, b1, b2, b3, b4 = newboxes(0, 0, 0, 0, 0)
     l0 = Lifetime(0, 4)

From pypy.commits at gmail.com  Sun Sep  3 03:12:45 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Sep 2017 00:12:45 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: test coalescing
Message-ID: <59abab6d.1b711c0a.685ea.5a92@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92307:5c1a449086cd
Date: 2017-09-03 08:57 +0200
http://bitbucket.org/pypy/pypy/changeset/5c1a449086cd/

Log:	test coalescing

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -37,12 +37,10 @@
         # For symmetrical operations, if 'y' is already in a register
         # and won't be used after the current operation finishes,
         # then swap the role of 'x' and 'y'
-        if self.longevity[x].last_usage > position:
-            if self.longevity[y].last_usage == position:
-                x, y = y, x
-                self.longevity.try_use_same_register(y, op)
-        else:
-            self.longevity.try_use_same_register(x, op)
+        if (self.longevity[x].last_usage > position and
+                self.longevity[y].last_usage == position):
+            x, y = y, x
+        self.longevity.try_use_same_register(x, op)
 
     def _consider_binop(self, op, position):
         self._consider_binop_part(op, position)
@@ -159,7 +157,6 @@
             return None
 
     def hint(self, position, args, argtypes, save_all_regs):
-        import pdb; pdb.set_trace()
         hinted_xmm = []
         hinted_gpr = []
         for i in range(len(args)):
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -1,4 +1,6 @@
-""" annoying integration tests for register allocation in the x86 backend """
+""" explicit integration tests for register allocation in the x86 backend """
+
+import pytest
 
 from rpython.jit.backend.llsupport.test import test_regalloc_integration
 from rpython.jit.backend.x86.assembler import Assembler386
@@ -29,7 +31,6 @@
         return Assembler386.mov(self, from_loc, to_loc)
 
     def regalloc_mov(self, from_loc, to_loc):
-        import pdb; pdb.set_trace()
         self._log("mov", from_loc, to_loc)
         return Assembler386.mov(self, from_loc, to_loc)
 
@@ -56,6 +57,10 @@
     def setup_method(self, meth):
         self.cpu.assembler._instr_log = self.log = []
 
+    def teardown_method(self, meth):
+        for l in self.log:
+            print l
+
     def test_call_use_correct_regs(self):
         ops = '''
         [i0, i1, i2, i3]
@@ -68,5 +73,38 @@
         '''
         self.interpret(ops, [5, 6, 7, 8])
         # two moves are needed from the stack frame to registers for arguments
-        # i0 and i1 one for the result to the stack
+        # i0 and i1, one for the result to the stack
         assert len([entry for entry in self.log if entry.name == "mov"]) == 3
+
+    def test_coalescing(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1)
+        i8 = int_add(i7, i3)
+        i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) []
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        # coalescing makes sure that i0 (and thus i71) lands in edi
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 2
+
+    @pytest.mark.skip("later")
+    def test_binop_dont_swap_unnecessarily(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1)
+        i8 = int_add(i2, 13)
+        i9 = int_add(i7, i8)
+        i10 = int_is_true(i9)
+        guard_true(i10) []
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        add1 = self.log[2]
+        op = self.log[5]
+        assert op.name == "op"
+        # make sure that the arguments of the third op are not swapped (since
+        # that would break coalescing between i7 and i9
+        assert op.args[1][0] is add1.args[-1]

From pypy.commits at gmail.com  Sun Sep  3 03:12:51 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Sep 2017 00:12:51 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: swap less,
 it can break coalescing
Message-ID: <59abab73.59451c0a.eb7a.8ef6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92310:2a1fd1c5aa14
Date: 2017-09-03 09:12 +0200
http://bitbucket.org/pypy/pypy/changeset/2a1fd1c5aa14/

Log:	swap less, it can break coalescing

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -505,14 +505,16 @@
     def _consider_binop_part(self, op, symm=False):
         x = op.getarg(0)
         y = op.getarg(1)
+        xloc = self.loc(x)
         argloc = self.loc(y)
-        #
-        # For symmetrical operations, if 'y' is already in a register
-        # and won't be used after the current operation finishes,
-        # then swap the role of 'x' and 'y'
-        # XXX only do that if x is *not* in a register
-        if (symm and isinstance(argloc, RegLoc) and
-                self.rm.longevity[y].last_usage == self.rm.position):
+
+        # For symmetrical operations, if x is not in a reg, but y is,
+        # and if x lives longer than the current operation while y dies, then
+        # swap the role of 'x' and 'y'
+        if (symm and not isinstance(xloc, RegLoc) and
+                isinstance(argloc, RegLoc) and
+                self.rm.longevity[x].last_usage > self.rm.position and
+                self.longevity[y].last_usage == self.rm.position):
             x, y = y, x
             argloc = self.loc(y)
         #
@@ -993,6 +995,8 @@
             self.assembler.test_location(resloc)
             self.assembler.guard_success_cc = rx86.Conditions['Z']
 
+        if not we_are_translated():
+            self.assembler.dump('%s <- %s(%s)' % (resloc, op, arglocs))
         self.assembler.cond_call(gcmap, imm_func, arglocs, resloc)
 
     consider_cond_call_value_i = consider_cond_call
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -102,7 +102,6 @@
         # coalescing makes sure that i0 (and thus i71) lands in edi
         assert len([entry for entry in self.log if entry.name == "mov"]) == 2
 
-    @pytest.mark.skip("later")
     def test_binop_dont_swap_unnecessarily(self):
         ops = '''
         [i0, i1, i2, i3]

From pypy.commits at gmail.com  Sun Sep  3 03:12:47 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Sep 2017 00:12:47 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: add fixed positions of a
 var to repr
Message-ID: <59abab6f.d24a1c0a.ccfd4.a13c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92308:b29a80fa1544
Date: 2017-09-03 08:58 +0200
http://bitbucket.org/pypy/pypy/changeset/b29a80fa1544/

Log:	add fixed positions of a var to repr

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -886,7 +886,11 @@
             assert self.definition_pos < min(self.real_usages)
 
     def __repr__(self):
-        return "%s:%s(%s)" % (self.definition_pos, self.real_usages, self.last_usage)
+        if self.fixed_positions:
+            s = " " + ", ".join("@%s in %s" % (index, reg) for (index, reg) in self.fixed_positions)
+        else:
+            s = ""
+        return "%s:%s(%s)%s" % (self.definition_pos, self.real_usages, self.last_usage, s)
 
 
 class FixedRegisterPositions(object):

From pypy.commits at gmail.com  Sun Sep  3 03:12:49 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sun, 03 Sep 2017 00:12:49 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: don't crash with unused
 operation
Message-ID: <59abab71.6881df0a.113ad.1835@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92309:2534cf818d52
Date: 2017-09-03 09:01 +0200
http://bitbucket.org/pypy/pypy/changeset/2534cf818d52/

Log:	don't crash with unused operation

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -24,6 +24,8 @@
         self.longevity = longevity
         for i in range(len(operations)):
             op = operations[i]
+            if rop.has_no_side_effect(op.opnum) and op not in self.longevity:
+                continue
             oplist[op.getopnum()](self, op, i)
 
     def not_implemented_op(self, op, position):
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -61,6 +61,18 @@
         for l in self.log:
             print l
 
+    def test_unused(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1) # unused
+        i9 = int_add(i2, i3)
+        finish(i9)
+        '''
+        # does not crash
+        self.interpret(ops, [5, 6, 7, 8])
+        assert len([entry for entry in self.log if entry.args[0] == "int_add"]) == 1
+
+
     def test_call_use_correct_regs(self):
         ops = '''
         [i0, i1, i2, i3]

From pypy.commits at gmail.com  Sun Sep  3 06:41:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 03 Sep 2017 03:41:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <59abdc53.4dce1c0a.d04ad.062d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92311:87310894a32d
Date: 2017-09-03 11:40 +0100
http://bitbucket.org/pypy/pypy/changeset/87310894a32d/

Log:	fix translation

diff --git a/pypy/module/cpyext/modsupport.py b/pypy/module/cpyext/modsupport.py
--- a/pypy/module/cpyext/modsupport.py
+++ b/pypy/module/cpyext/modsupport.py
@@ -153,7 +153,7 @@
             execf = rffi.cast(execfunctype, cur_slot[0].c_value)
             res = generic_cpy_call_dont_convert_result(space, execf, w_mod)
             state = space.fromcache(State)
-            if res:
+            if rffi.cast(lltype.Signed, res):
                 state.check_and_raise_exception()
                 raise oefmt(space.w_SystemError,
                             "execution of module %S failed without "

From pypy.commits at gmail.com  Sun Sep  3 11:44:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 03 Sep 2017 08:44:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59ac236d.1686df0a.7cafc.516b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92312:965ce7010d54
Date: 2017-09-03 16:44 +0100
http://bitbucket.org/pypy/pypy/changeset/965ce7010d54/

Log:	hg merge default

diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -141,6 +141,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = buffer(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FFL) << 56) | \
+               ((value & 0x000000000000FF00L) << 40) | \
+               ((value & 0x0000000000FF0000L) << 24) | \
+               ((value & 0x00000000FF000000L) << 8) | \
+               ((value & 0x000000FF00000000L) >> 8) | \
+               ((value & 0x0000FF0000000000L) >> 24) | \
+               ((value & 0x00FF000000000000L) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -265,6 +313,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -146,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -240,18 +241,26 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData, metaclass=StructOrUnionMeta):
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,8 +119,15 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian and Ubuntu, this is the command to install all build-time
-dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -548,6 +548,11 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
+* In PyPy, module and class dictionaries are optimized under the assumption
+  that deleting attributes from them are rare. Because of this, e.g.
+  ``del foo.bar`` where ``foo`` is a module (or class) that contains the
+  function ``bar``, is significantly slower than CPython.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -366,3 +366,56 @@
             )
         excinfo = raises(RecursionError, module.call_recursive)
         assert 'while calling recurse' in str(excinfo.value)
+
+    def test_build_class(self):
+            # make sure PyObject_Call generates a proper PyTypeObject,
+            # along the way verify that userslot has iter and next
+            module = self.import_extension('foo', [
+                ("object_call", "METH_O",
+                 '''
+                    return PyObject_Call((PyObject*)&PyType_Type, args, NULL);
+                 '''),
+                ('iter', "METH_O",
+                 '''
+                    if (NULL == args->ob_type->tp_iter)
+                    {
+                        PyErr_SetString(PyExc_TypeError, "NULL tp_iter");
+                        return NULL;
+                    }
+                    return args->ob_type->tp_iter(args);
+                 '''),
+                ('next', "METH_O",
+                 '''
+                    if (NULL == args->ob_type->tp_iternext)
+                    {
+                        PyErr_SetString(PyExc_TypeError, "NULL tp_iternext");
+                        return NULL;
+                    }
+                    return args->ob_type->tp_iternext(args);
+                 '''),])
+            def __init__(self, N):
+                self.N = N
+                self.i = 0
+
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                if self.i < self.N:
+                    i = self.i
+                    self.i += 1
+                    return i
+                raise StopIteration
+
+            d = {'__init__': __init__, '__iter__': __iter__, 'next': __next__,
+                 '__next__': __next__}
+            C = module.object_call(('Iterable', (object,), d))
+            c = C(5)
+            i = module.iter(c)
+            out = []
+            try:
+                while 1:
+                    out.append(module.next(i))
+            except StopIteration:
+                pass
+            assert out == [0, 1, 2, 3, 4]
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -311,13 +311,18 @@
                 setattr(pto, slot_names[0], slot_func_helper)
         elif ((w_type is space.w_list or w_type is space.w_tuple) and
               slot_names[0] == 'c_tp_as_number'):
-            # XXX hack - hwo can we generalize this? The problem is method
+            # XXX hack - how can we generalize this? The problem is method
             # names like __mul__ map to more than one slot, and we have no
             # convenient way to indicate which slots CPython have filled
             #
             # We need at least this special case since Numpy checks that
             # (list, tuple) do __not__ fill tp_as_number
             pass
+        elif ((space.issubtype_w(w_type, space.w_bytes) or
+                space.issubtype_w(w_type, space.w_unicode)) and
+                slot_names[0] == 'c_tp_as_number'):
+            # like above but for any str type
+            pass
         else:
             assert len(slot_names) == 2
             struct = getattr(pto, slot_names[0])
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -122,3 +122,11 @@
     else:
         space.delete(w_self, w_obj)
     return 0
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_iter(space, w_self):
+    return space.iter(w_self)
+
+ at slot_function([PyObject], PyObject)
+def slot_tp_iternext(space, w_self):
+    return space.next(w_self)
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_structures.py
@@ -22,7 +22,6 @@
         assert X._fields_ == [("a", c_int)]
         assert Y._fields_ == [("b", c_int)]
         assert Z._fields_ == [("a", c_int)]
-
         assert Y._names_ == ['a', 'b']
 
     def test_subclass_delayed(self):
@@ -455,6 +454,39 @@
         p = pointer(obj)
         assert p.contents._b_base_ is p
 
+    def test_swapped_bytes(self):
+        import sys
+
+        for i in [c_short, c_int, c_long, c_longlong,
+                  c_float, c_double, c_ushort, c_uint,
+                  c_ulong, c_ulonglong]:
+            FIELDS = [
+                ('n', i)
+            ]
+
+            class Native(Structure):
+                _fields_ = FIELDS
+
+            class Big(BigEndianStructure):
+                _fields_ = FIELDS
+
+            class Little(LittleEndianStructure):
+                _fields_ = FIELDS
+
+            def dostruct(c):
+                ba = create_string_buffer(sizeof(c))
+                ms = c.from_buffer(ba)
+                ms.n = 0xff00
+                return repr(ba[:])
+
+            if sys.byteorder == 'little':
+                assert dostruct(Native) == dostruct(Little)
+                assert dostruct(Native) != dostruct(Big)
+            else:
+                assert dostruct(Native) == dostruct(Big)
+                assert dostruct(Native) != dostruct(Little)
+
+
 class TestPointerMember(BaseCTypesTestChecker):
     def test_1(self):
         # a Structure with a POINTER field
diff --git a/pypy/objspace/std/sliceobject.py b/pypy/objspace/std/sliceobject.py
--- a/pypy/objspace/std/sliceobject.py
+++ b/pypy/objspace/std/sliceobject.py
@@ -132,6 +132,18 @@
         else:
             return space.w_False
 
+    def descr_ne(self, space, w_other):
+        if space.is_w(self, w_other):
+            return space.w_False
+        if not isinstance(w_other, W_SliceObject):
+            return space.w_NotImplemented
+        if space.eq_w(self.w_start, w_other.w_start) and \
+           space.eq_w(self.w_stop, w_other.w_stop) and \
+           space.eq_w(self.w_step, w_other.w_step):
+            return space.w_False
+        else:
+            return space.w_True
+
     def descr_lt(self, space, w_other):
         if space.is_w(self, w_other):
             return space.w_False   # see comments in descr_eq()
@@ -179,6 +191,7 @@
     __reduce__ = gateway.interp2app(W_SliceObject.descr__reduce__),
 
     __eq__ = gateway.interp2app(W_SliceObject.descr_eq),
+    __ne__ = gateway.interp2app(W_SliceObject.descr_ne),
     __lt__ = gateway.interp2app(W_SliceObject.descr_lt),
 
     start = slicewprop('w_start'),
diff --git a/pypy/objspace/std/test/test_sliceobject.py b/pypy/objspace/std/test/test_sliceobject.py
--- a/pypy/objspace/std/test/test_sliceobject.py
+++ b/pypy/objspace/std/test/test_sliceobject.py
@@ -94,6 +94,7 @@
         slice1 = slice(1, 2, 3)
         slice2 = slice(1, 2, 3)
         assert slice1 == slice2
+        assert not slice1 != slice2
         slice2 = slice(1, 2)
         assert slice1 != slice2
 
diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py
--- a/rpython/flowspace/test/test_objspace.py
+++ b/rpython/flowspace/test/test_objspace.py
@@ -839,15 +839,15 @@
             return x[s]
         graph = self.codetest(myfunc)
 
+    @py.test.mark.xfail
     def test_unichr_constfold(self):
-        py.test.skip("not working")
         def myfunc():
             return unichr(1234)
         graph = self.codetest(myfunc)
         assert graph.startblock.exits[0].target is graph.returnblock
 
+    @py.test.mark.xfail
     def test_unicode_constfold(self):
-        py.test.skip("not working for now")
         def myfunc():
             return unicode("1234")
         graph = self.codetest(myfunc)
diff --git a/rpython/jit/metainterp/optimizeopt/rewrite.py b/rpython/jit/metainterp/optimizeopt/rewrite.py
--- a/rpython/jit/metainterp/optimizeopt/rewrite.py
+++ b/rpython/jit/metainterp/optimizeopt/rewrite.py
@@ -10,7 +10,7 @@
 from rpython.jit.metainterp.optimizeopt.info import INFO_NONNULL, INFO_NULL
 from rpython.jit.metainterp.optimizeopt.util import _findall, make_dispatcher_method
 from rpython.jit.metainterp.resoperation import rop, ResOperation, opclasses,\
-     OpHelpers
+     OpHelpers, AbstractResOp
 from rpython.rlib.rarithmetic import highest_bit
 from rpython.rtyper.lltypesystem import llmemory
 from rpython.rtyper import rclass
@@ -490,6 +490,11 @@
 
     def postprocess_GUARD_TRUE(self, op):
         box = self.get_box_replacement(op.getarg(0))
+        if (isinstance(box, AbstractResOp) and
+                box.getopnum() == rop.INT_IS_TRUE):
+            # we can't use the (current) range analysis for this because
+            # "anything but 0" is not a valid range
+            self.pure_from_args(rop.INT_IS_ZERO, [box.getarg(0)], CONST_0)
         self.make_constant(box, CONST_1)
 
     def optimize_GUARD_FALSE(self, op):
@@ -497,6 +502,11 @@
 
     def postprocess_GUARD_FALSE(self, op):
         box = self.get_box_replacement(op.getarg(0))
+        if (isinstance(box, AbstractResOp) and
+                box.getopnum() == rop.INT_IS_ZERO):
+            # we can't use the (current) range analysis for this because
+            # "anything but 0" is not a valid range
+            self.pure_from_args(rop.INT_IS_TRUE, [box.getarg(0)], CONST_1)
         self.make_constant(box, CONST_0)
 
     def optimize_ASSERT_NOT_NONE(self, op):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizebasic.py
@@ -288,7 +288,6 @@
         self.optimize_loop(ops, expected)
 
     def test_int_is_true_is_zero(self):
-        py.test.skip("XXX implement me")
         ops = """
         [i0]
         i1 = int_is_true(i0)
@@ -305,6 +304,22 @@
         """
         self.optimize_loop(ops, expected)
 
+        ops = """
+        [i0]
+        i2 = int_is_zero(i0)
+        guard_false(i2) []
+        i1 = int_is_true(i0)
+        guard_true(i1) []
+        jump(i0)
+        """
+        expected = """
+        [i0]
+        i2 = int_is_zero(i0)
+        guard_false(i2) []
+        jump(i0)
+        """
+        self.optimize_loop(ops, expected)
+
     def test_int_is_zero_int_is_true(self):
         ops = """
         [i0]
diff --git a/rpython/jit/metainterp/resoperation.py b/rpython/jit/metainterp/resoperation.py
--- a/rpython/jit/metainterp/resoperation.py
+++ b/rpython/jit/metainterp/resoperation.py
@@ -400,7 +400,7 @@
         return rop.can_raise(self.getopnum())
 
     def is_foldable_guard(self):
-        return rop.is_foldable_guard(self.getopnun())
+        return rop.is_foldable_guard(self.getopnum())
 
     def is_primitive_array_access(self):
         """ Indicates that this operations loads/stores a
diff --git a/rpython/jit/metainterp/test/test_bridgeopt.py b/rpython/jit/metainterp/test/test_bridgeopt.py
--- a/rpython/jit/metainterp/test/test_bridgeopt.py
+++ b/rpython/jit/metainterp/test/test_bridgeopt.py
@@ -76,8 +76,11 @@
 
 
 box_strategy = strategies.builds(InputArgInt) | strategies.builds(InputArgRef)
-tuples = strategies.tuples(box_strategy, strategies.booleans()).filter(
-        lambda (box, known_class): isinstance(box, InputArgRef) or not known_class)
+def _make_tup(box, known_class):
+    if isinstance(box, InputArgInt):
+        known_class = False
+    return box, known_class
+tuples = strategies.builds(_make_tup, box_strategy, strategies.booleans())
 boxes_known_classes = strategies.lists(tuples, min_size=1)
 
 @given(boxes_known_classes)
diff --git a/rpython/translator/sandbox/rsandbox.py b/rpython/translator/sandbox/rsandbox.py
--- a/rpython/translator/sandbox/rsandbox.py
+++ b/rpython/translator/sandbox/rsandbox.py
@@ -27,17 +27,20 @@
 ll_read_not_sandboxed = rposix.external('read',
                                         [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
                                         rffi.SIZE_T,
-                                        sandboxsafe=True)
+                                        sandboxsafe=True,
+                                        _nowrapper=True)
 
 ll_write_not_sandboxed = rposix.external('write',
                                          [rffi.INT, rffi.CCHARP, rffi.SIZE_T],
                                          rffi.SIZE_T,
-                                         sandboxsafe=True)
+                                         sandboxsafe=True,
+                                         _nowrapper=True)
 
 
 @signature(types.int(), types.ptr(rffi.CCHARP.TO), types.int(),
     returns=types.none())
 def writeall_not_sandboxed(fd, buf, length):
+    fd = rffi.cast(rffi.INT, fd)
     while length > 0:
         size = rffi.cast(rffi.SIZE_T, length)
         count = rffi.cast(lltype.Signed, ll_write_not_sandboxed(fd, buf, size))
@@ -58,7 +61,8 @@
         buflen = self.buflen
         with lltype.scoped_alloc(rffi.CCHARP.TO, buflen) as buf:
             buflen = rffi.cast(rffi.SIZE_T, buflen)
-            count = ll_read_not_sandboxed(self.fd, buf, buflen)
+            fd = rffi.cast(rffi.INT, self.fd)
+            count = ll_read_not_sandboxed(fd, buf, buflen)
             count = rffi.cast(lltype.Signed, count)
             if count <= 0:
                 raise IOError

From pypy.commits at gmail.com  Sun Sep  3 11:58:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 03 Sep 2017 08:58:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport rpython changes from branch
 py3.5
Message-ID: <59ac268f.42b4df0a.dc365.f71f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92313:6a37749771cf
Date: 2017-09-03 16:57 +0100
http://bitbucket.org/pypy/pypy/changeset/6a37749771cf/

Log:	Backport rpython changes from branch py3.5

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem.rffi import CCHARP
+from rpython.rlib import jit
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform as target_platform
 
@@ -190,6 +191,8 @@
 
 IPX_TYPE
 
+SCM_RIGHTS
+
 POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL
 POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG
 
@@ -260,6 +263,7 @@
 sockaddr_ptr = lltype.Ptr(lltype.ForwardReference())
 addrinfo_ptr = lltype.Ptr(lltype.ForwardReference())
 
+
 # struct types
 CConfig.sockaddr = platform.Struct('struct sockaddr',
                                              [('sa_family', rffi.INT),
@@ -343,6 +347,650 @@
                                 [('ifr_ifindex', rffi.INT),
                                  ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
+# insert handler for sendmsg / recvmsg here
+if _POSIX:
+    includes = ['stddef.h',
+                'sys/socket.h',
+                'unistd.h',
+                'string.h',
+                'stdlib.h',
+                'errno.h',
+                'limits.h',
+                'stdio.h',
+                'sys/types.h',
+                'netinet/in.h',
+                'arpa/inet.h']
+    separate_module_sources = ['''
+
+        // special defines for returning from recvmsg
+        #define BAD_MSG_SIZE_GIVEN -10000
+        #define BAD_ANC_SIZE_GIVEN -10001
+        #define MAL_ANC -10002
+
+        // special defines for returning from sendmsg
+        #define MUL_MSGS_NOT_SUP -1000
+        #define ANC_DATA_TOO_LARGE -1001
+        #define ANC_DATA_TOO_LARGEX -1002
+
+        /*
+            Even though you could, theoretically, receive more than one message, IF you set the socket option,
+            CPython has hardcoded the message number to 1, and implemented the option to receive more then 1 in a
+            different socket method: recvmsg_into
+        */
+        #define MSG_IOVLEN 1 // CPython has hardcoded this as well.
+        #if INT_MAX > 0x7fffffff
+            #define SOCKLEN_T_LIMIT 0x7fffffff
+        #else
+        #define SOCKLEN_T_LIMIT INT_MAX
+        #endif
+
+        // ################################################################################################
+        // Recvmsg implementation and associated functions
+
+        // Taken from CPython. Determines the minimum memory space required for the ancillary data.
+        #ifdef CMSG_SPACE
+        static int
+        cmsg_min_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t space)
+        {
+            size_t cmsg_offset;
+            static const size_t cmsg_len_end = (offsetof(struct cmsghdr, cmsg_len) +
+                                                sizeof(cmsgh->cmsg_len));
+
+            /* Note that POSIX allows msg_controllen to be of signed type. */
+            if (cmsgh == NULL || msg->msg_control == NULL)
+                return 0;
+            /* Note that POSIX allows msg_controllen to be of a signed type. This is
+               annoying under OS X as it's unsigned there and so it triggers a
+               tautological comparison warning under Clang when compared against 0.
+               Since the check is valid on other platforms, silence the warning under
+               Clang. */
+            #ifdef __clang__
+            #pragma clang diagnostic push
+            #pragma clang diagnostic ignored "-Wtautological-compare"
+            #endif
+            #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+            #pragma GCC diagnostic push
+            #pragma GCC diagnostic ignored "-Wtype-limits"
+            #endif
+            if (msg->msg_controllen < 0)
+                return 0;
+            #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+            #pragma GCC diagnostic pop
+            #endif
+            #ifdef __clang__
+            #pragma clang diagnostic pop
+            #endif
+            if (space < cmsg_len_end)
+                space = cmsg_len_end;
+            cmsg_offset = (char *)cmsgh - (char *)msg->msg_control;
+            return (cmsg_offset <= (size_t)-1 - space &&
+                    cmsg_offset + space <= msg->msg_controllen);
+        }
+        #endif
+
+        // Taken from CPython.
+        #ifdef CMSG_LEN
+        /* If pointer CMSG_DATA(cmsgh) is in buffer msg->msg_control, set
+           *space to number of bytes following it in the buffer and return
+           true; otherwise, return false.  Assumes cmsgh, msg->msg_control and
+           msg->msg_controllen are valid. */
+        static int
+        get_cmsg_data_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *space)
+        {
+            size_t data_offset;
+            char *data_ptr;
+
+            if ((data_ptr = (char *)CMSG_DATA(cmsgh)) == NULL)
+                return 0;
+            data_offset = data_ptr - (char *)msg->msg_control;
+            if (data_offset > msg->msg_controllen)
+                return 0;
+            *space = msg->msg_controllen - data_offset;
+            return 1;
+        }
+
+        // Taken from CPython.
+        /* If cmsgh is invalid or not contained in the buffer pointed to by
+           msg->msg_control, return -1.  If cmsgh is valid and its associated
+           data is entirely contained in the buffer, set *data_len to the
+           length of the associated data and return 0.  If only part of the
+           associated data is contained in the buffer but cmsgh is otherwise
+           valid, set *data_len to the length contained in the buffer and
+           return 1. */
+        static int
+        get_cmsg_data_len(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *data_len)
+        {
+            size_t space, cmsg_data_len;
+
+            if (!cmsg_min_space(msg, cmsgh, CMSG_LEN(0)) ||
+                cmsgh->cmsg_len < CMSG_LEN(0))
+                return -1;
+            cmsg_data_len = cmsgh->cmsg_len - CMSG_LEN(0);
+            if (!get_cmsg_data_space(msg, cmsgh, &space))
+                return -1;
+            if (space >= cmsg_data_len) {
+                *data_len = cmsg_data_len;
+                return 0;
+            }
+            *data_len = space;
+            return 1;
+        }
+        #endif    /* CMSG_LEN */
+
+        /*
+            Structure meant to hold the information received after a recvmsg is performed.
+            Essentially it holds: the address, the message, the ancillary data and the return flags.
+            I use this structure for 2 main reasons:
+               - keep things ordered
+               - some of the ancillary parameters need to be int not long (rffi SignedP is actually long*),
+                 therefore I cannot use the parameters directly
+        */
+        struct recvmsg_info
+        {
+            struct sockaddr* address; // address fields
+            socklen_t addrlen;
+            int* length_of_messages; // message fields
+            char** messages;
+            int no_of_messages;
+            int size_of_ancillary; // ancillary fields
+            int* levels;
+            int* types;
+            char** file_descr;
+            int* descr_per_ancillary;
+            int retflag; // return flag field
+        };
+
+        /*
+            Wrapper function over recvmsg. Since it returns a lot of data,
+            in a structure that is hard to parse in rffi, it was implemented in C.
+            All the parameters, save the socket fd, message_size, ancillary_size
+            will be malloc'd and/or modified.
+        */
+        RPY_EXTERN
+        int recvmsg_implementation(
+                                  int socket_fd,
+                                  int message_size,
+                                  int ancillary_size,
+                                  int flags,
+                                  struct sockaddr* address,
+                                  socklen_t* addrlen,
+                                  long** length_of_messages,
+                                  char** messages,
+                                  long* no_of_messages,
+                                  long* size_of_ancillary,
+                                  long** levels,
+                                  long** types,
+                                  char** file_descr,
+                                  long** descr_per_ancillary,
+                                  long* retflag)
+
+        {
+
+            struct sockaddr* recvd_address;
+            socklen_t recvd_addrlen;
+            struct msghdr msg = {0};
+            void *controlbuf = NULL;
+            struct cmsghdr *cmsgh;
+            int cmsg_status;
+            struct iovec iov;
+            struct recvmsg_info* retinfo;
+            int error_flag;   // variable to be set in case of special errors.
+            int cmsgdatalen = 0;
+
+            // variables that are set to 1, if the message charp has been allocated
+            // and if the ancillary variables have been allocated. To be used in case of failure.
+            int iov_alloc = 0;
+            int anc_alloc = 0;
+
+            retinfo = (struct recvmsg_info*) malloc(sizeof(struct recvmsg_info));
+
+            if (ancillary_size > SOCKLEN_T_LIMIT){
+                error_flag = BAD_ANC_SIZE_GIVEN;
+                goto fail;
+            }
+
+            // Setup the messages iov struct memory
+            iov.iov_base = (char*) malloc(message_size);
+            memset(iov.iov_base, 0, message_size);
+            iov.iov_len = message_size;
+
+            // Setup the ancillary buffer memory
+            controlbuf = malloc(ancillary_size);
+
+            // Setup the recv address memory
+            recvd_addrlen = sizeof(struct sockaddr_storage);
+            recvd_address = (struct sockaddr*) malloc(recvd_addrlen);
+
+            memset(recvd_address, 0,recvd_addrlen);
+
+            // Setup the msghdr struct
+            msg.msg_name = recvd_address;
+            msg.msg_namelen = recvd_addrlen;
+            msg.msg_iov = &iov;
+            msg.msg_iovlen = MSG_IOVLEN;
+            msg.msg_control = controlbuf;
+            msg.msg_controllen = ancillary_size;
+
+            // Link my structure to the msghdr fields
+            retinfo->address = msg.msg_name;
+            retinfo->length_of_messages = (int*) malloc (MSG_IOVLEN * sizeof(int));
+            retinfo->no_of_messages = MSG_IOVLEN;
+            retinfo->messages = (char**) malloc (MSG_IOVLEN * sizeof(char*));
+            retinfo->messages[0] = msg.msg_iov->iov_base;
+
+            iov_alloc = 1;
+            ssize_t bytes_recvd = 0;
+
+            bytes_recvd = recvmsg(socket_fd, &msg, flags);
+
+            if (bytes_recvd < 0){
+                 goto fail;
+            }
+
+            retinfo->addrlen = (socklen_t) msg.msg_namelen;
+            retinfo->length_of_messages[0] = msg.msg_iov->iov_len;
+
+            // Count the ancillary items & allocate the memory
+            int anc_counter = 0;
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+
+                 anc_counter++;
+            }
+            retinfo->size_of_ancillary = anc_counter;
+            retinfo->file_descr = (char**) malloc (anc_counter * sizeof(char*));
+            retinfo->levels = (int*) malloc(anc_counter * sizeof(int));
+            retinfo->types = (int*) malloc(anc_counter * sizeof(int));
+            retinfo->descr_per_ancillary = (int*) malloc(anc_counter * sizeof(int));
+            anc_alloc = 1;
+
+            // Extract the ancillary items
+            int i=0;
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+                 size_t local_size = 0;
+                 cmsg_status = get_cmsg_data_len(&msg, cmsgh, &local_size);
+                 if (cmsg_status !=0 ){
+                    error_flag = MAL_ANC;
+                    goto err_closefds;
+                 }
+                 retinfo->file_descr[i] = (char*) malloc(local_size);
+                 memcpy(retinfo->file_descr[i], CMSG_DATA(cmsgh), local_size);
+                 retinfo->levels[i] = cmsgh->cmsg_level;
+                 retinfo->types[i] = cmsgh->cmsg_type;
+                 retinfo->descr_per_ancillary[i] =local_size;
+                 i++;
+
+            }
+            retinfo->retflag = msg.msg_flags;
+
+            // Set the parameters of address
+            memcpy(address,retinfo->address,retinfo->addrlen);
+            *addrlen = retinfo->addrlen;
+
+            // Set the parameters of message
+            no_of_messages[0] = retinfo->no_of_messages;
+            size_of_ancillary[0] = retinfo->size_of_ancillary;
+            *length_of_messages = (long*) malloc (sizeof(long) * retinfo->no_of_messages);
+            //memcpy(*length_of_messages, retinfo->length_of_messages, sizeof(int) * retinfo->no_of_messages);
+            int counter = 0;
+            for (i=0; i< retinfo->no_of_messages; i++){
+                counter += retinfo->length_of_messages[i];
+                length_of_messages[0][i] = retinfo->length_of_messages[i];
+            }
+            memset(*messages, 0, sizeof(char) * counter);
+            counter = 0;
+            for(i=0; i< retinfo->no_of_messages; i++){
+                memcpy(*messages+counter,retinfo->messages[i],retinfo->length_of_messages[i]);
+                counter += retinfo->length_of_messages[i];
+            }
+
+            // Set the parameters of ancillary
+            *levels = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            *types = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            *descr_per_ancillary = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            counter = 0;
+            for (i=0; i < retinfo->size_of_ancillary; i++){
+                counter += retinfo->descr_per_ancillary[i];
+                // Convert the int* to long*
+                levels[0][i] = (long) retinfo->levels[i];
+                types[0][i] = (long) retinfo->types[i];
+                descr_per_ancillary[0][i] = (long) retinfo->descr_per_ancillary[i];
+            }
+            *file_descr = (char*) malloc (sizeof(char) * counter);
+            memset(*file_descr, 0, sizeof(char) * counter);
+            counter = 0;
+            for (i=0; i<retinfo->size_of_ancillary; i++){
+                memcpy(*file_descr+counter,retinfo->file_descr[i], retinfo->descr_per_ancillary[i]);
+                counter += retinfo->descr_per_ancillary[i];
+            }
+
+            // Set the retflag
+            retflag[0] = retinfo->retflag;
+
+            // Free the memory
+            free(retinfo->address);
+            free(retinfo->length_of_messages);
+            free(retinfo->levels);
+            free(retinfo->types);
+            free(retinfo->descr_per_ancillary);
+            for(i = 0; i<retinfo->no_of_messages; i++)
+                free(retinfo->messages[i]);
+            for (i = 0; i < retinfo->size_of_ancillary; i++)
+                free(retinfo->file_descr[i]);
+            free(retinfo->file_descr);
+            free(retinfo->messages);
+            free(retinfo);
+            free(controlbuf);
+
+            return bytes_recvd;
+
+        fail:
+            if (anc_alloc){
+                free(retinfo->file_descr);
+                free(retinfo->levels);
+                free(retinfo->types);
+                free(retinfo->descr_per_ancillary);
+                free(retinfo->length_of_messages);
+                free(retinfo->messages[0]);
+                free(retinfo->messages);
+                free(retinfo->address);
+                free(retinfo);
+                free(controlbuf);
+
+            }else{
+                if (iov_alloc){
+                    free(retinfo->length_of_messages);
+                    free(retinfo->messages[0]);
+                    free(retinfo->messages);
+                    free(retinfo->address);
+                    free(controlbuf);
+                    free(retinfo);
+                }
+            }
+            return error_flag;
+
+        err_closefds:
+        // Special case for UNIX sockets. In case file descriptors are received, they need to be closed.
+        // Taken from CPython
+        #ifdef SCM_RIGHTS
+            /* Close all descriptors coming from SCM_RIGHTS, so they don't leak. */
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+                size_t dataleng;
+                cmsg_status = get_cmsg_data_len(&msg, cmsgh, &dataleng);
+                cmsgdatalen = (int) dataleng;
+                if (cmsg_status < 0)
+                    break;
+                if (cmsgh->cmsg_level == SOL_SOCKET &&
+                    cmsgh->cmsg_type == SCM_RIGHTS) {
+                    size_t numfds;
+                    int *fdp;
+
+                    numfds = cmsgdatalen / sizeof(int);
+                    fdp = (int *)CMSG_DATA(cmsgh);
+                    while (numfds-- > 0)
+                        close(*fdp++);
+                }
+                if (cmsg_status != 0)
+                    break;
+            }
+        #endif /* SCM_RIGHTS */
+            goto fail;
+        }
+
+
+        // ################################################################################################
+        // Sendmsg implementation and associated functions
+
+        #ifdef CMSG_LEN
+        static int
+        get_CMSG_LEN(size_t length, size_t *result)
+        {
+            size_t tmp;
+
+            if (length > (SOCKLEN_T_LIMIT - CMSG_LEN(0)))
+                return 0;
+            tmp = CMSG_LEN(length);
+            if ((tmp > SOCKLEN_T_LIMIT) || (tmp < length))
+                return 0;
+            *result = tmp;
+            return 1;
+        }
+        #endif
+
+        #ifdef CMSG_SPACE
+        /* If length is in range, set *result to CMSG_SPACE(length) and return
+           true; otherwise, return false. */
+        static int
+        get_CMSG_SPACE(size_t length, size_t *result)
+        {
+            size_t tmp;
+
+            /* Use CMSG_SPACE(1) here in order to take account of the padding
+               necessary before *and* after the data. */
+            if (length > (SOCKLEN_T_LIMIT - CMSG_SPACE(1)))
+                return 0;
+            tmp = CMSG_SPACE(length);
+            if ((tmp > SOCKLEN_T_LIMIT) || (tmp < length))
+                return 0;
+            *result = tmp;
+            return 1;
+        }
+        #endif
+
+        /*
+            sendmsg_implementation is a wrapper over sendmsg of the API.
+            It was inspired from the way CPython did their implementation of this.
+            The main reason that it was written in C, is the struct msghdr,
+            which contains the ancillary data in a linked list of cmsghdr structures.
+            It was simpler to use it in C, and then push the simpler types of data via rffi.
+        */
+        RPY_EXTERN
+        int sendmsg_implementation
+                            (int socket,
+                            struct sockaddr* address,
+                            socklen_t addrlen,
+                            long* length_of_messages,
+                            char** messages,
+                            int no_of_messages,
+                            long* levels,
+                            long* types,
+                            char** file_descriptors,
+                            long* no_of_fds,
+                            int control_length,
+                            int flag
+                            )
+        {
+
+            struct msghdr        msg = {0};
+            struct cmsghdr       *cmsg;
+            void* controlbuf = NULL;
+            int retval;
+            size_t i;
+
+            // Prepare the msghdr structure for the send:
+
+            // Add the address
+            if (address != NULL) {
+                msg.msg_name = address;
+                msg.msg_namelen = addrlen;
+            }
+
+            // Add the message
+            struct iovec *iovs = NULL;
+            if (no_of_messages > 0){
+
+                iovs = (struct iovec*) malloc(no_of_messages * sizeof(struct iovec));
+                memset(iovs, 0, no_of_messages * sizeof(struct iovec));
+                msg.msg_iov = iovs;
+                msg.msg_iovlen = no_of_messages;
+
+                for (i=0; i< no_of_messages; i++){
+                    iovs[i].iov_base = messages[i];
+                    iovs[i].iov_len = length_of_messages[i];
+                }
+            }
+
+            // Add the ancillary
+            #ifndef CMSG_SPACE
+                if (control_length > 1){
+                    free(iovs);
+                    return MUL_MSGS_NOT_SUP;
+                }
+            #endif
+            if (control_length > 0){
+
+                //compute the total size of the ancillary
+                //getting the exact amount of space can be tricky and os dependent.
+                size_t total_size_of_ancillary = 0;
+                size_t space;
+                size_t controllen = 0, controllen_last = 0;
+                for (i = 0; i< control_length; i++){
+                    total_size_of_ancillary = no_of_fds[i];
+                    #ifdef CMSG_SPACE
+                        if (!get_CMSG_SPACE(total_size_of_ancillary, &space)) {
+                    #else
+                        if (!get_CMSG_LEN(total_size_of_ancillary, &space)) {
+                    #endif
+                            if (iovs != NULL)
+                                free(iovs);
+                            return ANC_DATA_TOO_LARGE;
+                        }
+                    controllen +=space;
+                    if ((controllen > SOCKLEN_T_LIMIT) || (controllen < controllen_last)) {
+                        if (iovs != NULL)
+                                free(iovs);
+                        return ANC_DATA_TOO_LARGEX;
+                    }
+                    controllen_last = controllen;
+                }
+
+                controlbuf = malloc(controllen);
+                msg.msg_control= controlbuf;
+                msg.msg_controllen = controllen;
+
+                // memset controlbuf to 0 to avoid trash in the ancillary
+                memset(controlbuf, 0, controllen);
+                cmsg = NULL;
+                for (i = 0; i< control_length; i++){
+                    cmsg = (i == 0) ? CMSG_FIRSTHDR(&msg) : CMSG_NXTHDR(&msg, cmsg);
+
+                    cmsg->cmsg_level = (int) levels[i];
+                    cmsg->cmsg_type = (int) types[i];
+                    cmsg->cmsg_len = CMSG_LEN(sizeof(char) * no_of_fds[i]);
+                    memcpy(CMSG_DATA(cmsg), file_descriptors[i], sizeof(char) * no_of_fds[i]);
+                }
+
+
+            }
+            // Add the flags
+            msg.msg_flags = flag;
+
+            // Send the data
+            retval = sendmsg(socket, &msg, flag);
+
+            // free everything that was allocated here, and we would not need in rsocket
+            if (iovs != NULL)
+                free(iovs);
+            if (controlbuf !=NULL)
+               free(controlbuf);
+
+            return retval;
+        }
+
+        // ################################################################################################
+        // Wrappers for CMSG_SPACE and CMSG_LEN
+
+        /*
+            These 2 functions are wrappers over sys/socket.h's CMSG_SPACE and CMSG_LEN.
+            They are identical to CPython's.
+        */
+        #ifdef CMSG_SPACE
+        RPY_EXTERN
+        size_t CMSG_SPACE_wrapper(size_t desired_space){
+            size_t result;
+            if (!get_CMSG_SPACE(desired_space, &result)){
+                return 0;
+            }
+            return result;
+        }
+        #endif
+
+        #ifdef CMSG_LEN
+        RPY_EXTERN
+        size_t CMSG_LEN_wrapper(size_t desired_len){
+            size_t result;
+            if (!get_CMSG_LEN(desired_len, &result)){
+                return 0;
+            }
+            return result;
+        }
+        #endif
+
+        // ################################################################################################
+        // Extra functions that I needed
+
+        /*
+           This function is used to memcpy from a char* at an offset.
+           Could not get rffi.c_memcpy to do it at an offset, so I made my own.
+        */
+        RPY_EXTERN
+        int memcpy_from_CCHARP_at_offset_and_size(char* stringfrom, char** stringto, int offset, int size){
+            *stringto = memcpy(*stringto, stringfrom + offset, size);
+            return 0;
+        }
+
+        /*
+            These functions free memory that was allocated in C (sendmsg or recvmsg) was used in rsocket and now needs cleanup
+        */
+        RPY_EXTERN
+        int free_pointer_to_signedp(int** ptrtofree){
+            free(*ptrtofree);
+            return 0;
+        }
+
+        RPY_EXTERN
+        int free_ptr_to_charp(char** ptrtofree){
+            free(*ptrtofree);
+            return 0;
+        }
+
+    ''',]
+
+    post_include_bits =[ "RPY_EXTERN "
+                         "int sendmsg_implementation(int socket, struct sockaddr* address, socklen_t addrlen, long* length_of_messages, char** messages, int no_of_messages, long* levels, long* types, char** file_descriptors, long* no_of_fds, int control_length, int flag );\n"
+                         "RPY_EXTERN "
+                         "int recvmsg_implementation(int socket_fd, int message_size, int ancillary_size, int flags, struct sockaddr* address, socklen_t* addrlen, long** length_of_messages, char** messages, long* no_of_messages, long* size_of_ancillary, long** levels, long** types, char** file_descr, long** descr_per_ancillary, long* flag);\n"
+                         "static "
+                         "int cmsg_min_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t space);\n"
+                         "static "
+                         "int get_cmsg_data_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *space);\n"
+                         "static "
+                         "int get_cmsg_data_len(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *data_len);\n"
+                         "static "
+                         "int get_CMSG_LEN(size_t length, size_t *result);\n"
+                         "static "
+                         "int get_CMSG_SPACE(size_t length, size_t *result);\n"
+                         "RPY_EXTERN "
+                         "size_t CMSG_LEN_wrapper(size_t desired_len);\n"
+                         "RPY_EXTERN "
+                         "size_t CMSG_SPACE_wrapper(size_t desired_space);\n"
+                         "RPY_EXTERN "
+                         "int memcpy_from_CCHARP_at_offset_and_size(char* stringfrom, char** stringto, int offset, int size);\n"
+                         "RPY_EXTERN "
+                         "int free_pointer_to_signedp(int** ptrtofree);\n"
+                         "RPY_EXTERN "
+                         "int free_ptr_to_charp(char** ptrtofree);\n"
+                         ]
+
+
+    compilation_info = ExternalCompilationInfo(
+                                    includes=includes,
+                                    separate_module_sources=separate_module_sources,
+                                    post_include_bits=post_include_bits,
+                               )
+
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
     CConfig.WSANETWORKEVENTS = platform.Struct(
@@ -387,6 +1035,7 @@
 sockaddr_ptr.TO.become(cConfig.sockaddr)
 addrinfo_ptr.TO.become(cConfig.addrinfo)
 
+
 # fill in missing constants with reasonable defaults
 cConfig.NI_MAXHOST = cConfig.NI_MAXHOST or 1025
 cConfig.NI_MAXSERV = cConfig.NI_MAXSERV or 32
@@ -571,11 +1220,32 @@
 recvfrom = external('recvfrom', [socketfd_type, rffi.VOIDP, size_t,
                            rffi.INT, sockaddr_ptr, socklen_t_ptr], rffi.INT,
                     save_err=SAVE_ERR)
+recvmsg = jit.dont_look_inside(rffi.llexternal("recvmsg_implementation",
+                                               [rffi.INT, rffi.INT, rffi.INT, rffi.INT,sockaddr_ptr, socklen_t_ptr, rffi.SIGNEDPP, rffi.CCHARPP,
+                                                rffi.SIGNEDP,rffi.SIGNEDP, rffi.SIGNEDPP, rffi.SIGNEDPP, rffi.CCHARPP, rffi.SIGNEDPP, rffi.SIGNEDP],
+                                               rffi.INT, save_err=SAVE_ERR,
+                                               compilation_info=compilation_info))
+
+memcpy_from_CCHARP_at_offset = jit.dont_look_inside(rffi.llexternal("memcpy_from_CCHARP_at_offset_and_size",
+                                    [rffi.CCHARP, rffi.CCHARPP,rffi.INT,rffi.INT],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+freeccharp = jit.dont_look_inside(rffi.llexternal("free_ptr_to_charp",
+                                    [rffi.CCHARPP],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+freesignedp = jit.dont_look_inside(rffi.llexternal("free_pointer_to_signedp",
+                                    [rffi.SIGNEDPP],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+
 send = external('send', [socketfd_type, rffi.CCHARP, size_t, rffi.INT],
                        ssize_t, save_err=SAVE_ERR)
 sendto = external('sendto', [socketfd_type, rffi.VOIDP, size_t, rffi.INT,
                                     sockaddr_ptr, socklen_t], ssize_t,
                   save_err=SAVE_ERR)
+sendmsg = jit.dont_look_inside(rffi.llexternal("sendmsg_implementation",
+                               [rffi.INT, sockaddr_ptr, socklen_t, rffi.SIGNEDP, rffi.CCHARPP, rffi.INT,
+                                rffi.SIGNEDP, rffi.SIGNEDP, rffi.CCHARPP, rffi.SIGNEDP, rffi.INT, rffi.INT],
+                               rffi.INT, save_err=SAVE_ERR,
+                               compilation_info=compilation_info))
+CMSG_SPACE = jit.dont_look_inside(rffi.llexternal("CMSG_SPACE_wrapper",[size_t], size_t, save_err=SAVE_ERR,compilation_info=compilation_info))
+CMSG_LEN = jit.dont_look_inside(rffi.llexternal("CMSG_LEN_wrapper",[size_t], size_t, save_err=SAVE_ERR,compilation_info=compilation_info))
+
 socketshutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT,
                           save_err=SAVE_ERR)
 gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1312,9 +1312,17 @@
 @replace_os_function('link')
 @specialize.argtype(0, 1)
 def link(oldpath, newpath):
-    oldpath = _as_bytes0(oldpath)
-    newpath = _as_bytes0(newpath)
-    handle_posix_error('link', c_link(oldpath, newpath))
+    if not _WIN32:
+        oldpath = _as_bytes0(oldpath)
+        newpath = _as_bytes0(newpath)
+        handle_posix_error('link', c_link(oldpath, newpath))
+    else:
+        traits = _preferred_traits(oldpath)
+        win32traits = make_win32_traits(traits)
+        oldpath = traits.as_str0(oldpath)
+        newpath = traits.as_str0(newpath)
+        if not win32traits.CreateHardLink(newpath, oldpath, None):
+            raise rwin32.lastSavedWindowsError()
 
 @replace_os_function('symlink')
 @specialize.argtype(0, 1)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -963,6 +963,126 @@
                 return (read_bytes, address)
             raise self.error_handler()
 
+    @jit.dont_look_inside
+    def recvmsg(self, message_size, ancbufsize = 0, flags = 0):
+        """
+        Receive up to message_size bytes from a message. Also receives ancillary data.
+        Returns the message, ancillary, flag and address of the sender.
+        :param message_size: Maximum size of the message to be received
+        :param ancbufsize:  Maximum size of the ancillary data to be received
+        :param flags: Receive flag. For more details, please check the Unix manual
+        :return: a tuple consisting of the message, the ancillary data, return flag and the address.
+        """
+        if message_size < 0:
+            raise RSocketError("Invalid message size")
+        if ancbufsize < 0:
+            raise RSocketError("invalid ancillary data buffer length")
+
+        self.wait_for_data(False)
+        address, addr_p, addrlen_p  = self._addrbuf()
+        len_of_msgs = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        messages = lltype.malloc(rffi.CCHARPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        messages[0] = lltype.malloc(rffi.CCHARP.TO, message_size,flavor='raw',track_allocation=True,nonmovable=False)
+        rffi.c_memset(messages[0], 0, message_size)
+        no_of_messages  = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        no_of_messages[0] = rffi.cast(rffi.SIGNED, 0)
+        size_of_anc = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        size_of_anc[0] = rffi.cast(rffi.SIGNED,0)
+        levels = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        types = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        file_descr = lltype.malloc(rffi.CCHARPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        descr_per_anc = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        retflag = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        retflag[0] = rffi.cast(rffi.SIGNED,0)
+
+        # a mask for the SIGNEDP's that need to be cast to int. (long default)
+        reply = _c.recvmsg(self.fd, rffi.cast(lltype.Signed,message_size),
+                           rffi.cast(lltype.Signed,ancbufsize),rffi.cast(lltype.Signed,flags),
+                           addr_p, addrlen_p, len_of_msgs, messages, no_of_messages,size_of_anc,
+                           levels, types,file_descr,descr_per_anc,retflag)
+        if reply >= 0:
+            anc_size  = rffi.cast(rffi.SIGNED,size_of_anc[0])
+            returnflag  = rffi.cast(rffi.SIGNED,retflag[0])
+            addrlen = rffi.cast(rffi.SIGNED,addrlen_p[0])
+
+            retmsg = rffi.charpsize2str(messages[0],reply)
+
+            offset = 0
+            list_of_tuples = []
+
+            pre_anc = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw', track_allocation=True, nonmovable=False)
+            for i in range(anc_size):
+                level = rffi.cast(rffi.SIGNED, levels[0][i])
+                type = rffi.cast(rffi.SIGNED, types[0][i])
+                bytes_in_anc = rffi.cast(rffi.SIGNED, descr_per_anc[0][i])
+                pre_anc[0] = lltype.malloc(rffi.CCHARP.TO, bytes_in_anc,flavor='raw',track_allocation=True,nonmovable=False)
+                _c.memcpy_from_CCHARP_at_offset(file_descr[0], pre_anc,rffi.cast(rffi.SIGNED,offset), bytes_in_anc)
+                anc = rffi.charpsize2str(pre_anc[0],bytes_in_anc)
+                tup = (level,type, anc)
+                list_of_tuples.append(tup)
+                offset += bytes_in_anc
+                lltype.free(pre_anc[0], flavor='raw')
+
+            if addrlen:
+                address.addrlen = addrlen
+            else:
+                address.unlock()
+                address = None
+
+            rettup = (retmsg,list_of_tuples,returnflag,address)
+
+            if address is not None:
+                address.unlock()
+            # free underlying complexity first
+            _c.freeccharp(file_descr)
+            _c.freesignedp(len_of_msgs)
+            _c.freesignedp(levels)
+            _c.freesignedp(types)
+            _c.freesignedp(descr_per_anc)
+
+            lltype.free(messages[0], flavor='raw')
+            lltype.free(pre_anc,flavor='raw')
+            lltype.free(messages,flavor='raw')
+            lltype.free(file_descr,flavor='raw')
+            lltype.free(len_of_msgs,flavor='raw')
+            lltype.free(no_of_messages, flavor='raw')
+            lltype.free(size_of_anc, flavor='raw')
+            lltype.free(levels, flavor='raw')
+            lltype.free(types, flavor='raw')
+            lltype.free(descr_per_anc, flavor='raw')
+            lltype.free(retflag, flavor='raw')
+            lltype.free(addrlen_p,flavor='raw')
+
+            return rettup
+        else:
+
+            #in case of failure the underlying complexity has already been freed
+            lltype.free(messages[0], flavor='raw')
+            lltype.free(messages, flavor='raw')
+            lltype.free(file_descr, flavor='raw')
+            lltype.free(len_of_msgs, flavor='raw')
+            lltype.free(no_of_messages, flavor='raw')
+            lltype.free(size_of_anc, flavor='raw')
+            lltype.free(levels, flavor='raw')
+            lltype.free(types, flavor='raw')
+            lltype.free(descr_per_anc, flavor='raw')
+            lltype.free(retflag, flavor='raw')
+            lltype.free(addrlen_p, flavor='raw')
+
+            if address is not None:
+                address.unlock()
+            if _c.geterrno() == _c.EINTR:
+                raise last_error()
+            if (reply == -10000):
+                raise RSocketError("Invalid message size")
+            if (reply == -10001):
+                raise RSocketError("Invalid ancillary data buffer length")
+            if (reply == -10002):
+                raise RSocketError("received malformed or improperly truncated ancillary data")
+            raise last_error()
+
+
+
     def send_raw(self, dataptr, length, flags=0):
         """Send data from a CCHARP buffer."""
         self.wait_for_data(True)
@@ -1009,6 +1129,86 @@
             raise self.error_handler()
         return res
 
+    @jit.dont_look_inside
+    def sendmsg(self, messages, ancillary=None, flags=0, address=None):
+        """
+        Send data and ancillary on a socket. For use of ancillary data, please check the Unix manual.
+        Work on connectionless sockets via the address parameter.
+        :param messages: a message that is a list of strings
+        :param ancillary: data to be sent separate from the message body. Needs to be a list of tuples.
+                            E.g. [(level,type, bytes),...]. Default None.
+        :param flags: the flag to be set for sendmsg. Please check the Unix manual regarding values. Default 0
+        :param address: address of the recepient. Useful for when sending on connectionless sockets. Default None
+        :return: Bytes sent from the message
+        """
+        need_to_free_address = True
+        if address is None:
+            need_to_free_address = False
+            addr = lltype.nullptr(_c.sockaddr)
+            addrlen = 0
+        else:
+            addr = address.lock()
+            addrlen = address.addrlen
+
+        no_of_messages = len(messages)
+        messages_ptr = lltype.malloc(rffi.CCHARPP.TO,no_of_messages+1,flavor='raw',track_allocation=True,nonmovable=False)
+        messages_length_ptr = lltype.malloc(rffi.SIGNEDP.TO,no_of_messages,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        counter = 0
+        for message in messages:
+            messages_ptr[counter] = rffi.str2charp(message)
+            messages_length_ptr[counter] = rffi.cast(rffi.SIGNED, len(message))
+            counter += 1
+        messages_ptr[counter] = lltype.nullptr(rffi.CCHARP.TO)
+        if ancillary is not None:
+            size_of_ancillary = len(ancillary)
+        else:
+            size_of_ancillary = 0
+        levels = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        types = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        desc_per_ancillary = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        file_descr = lltype.malloc(rffi.CCHARPP.TO, size_of_ancillary,flavor='raw', track_allocation=True,nonmovable=False)
+        if ancillary is not None:
+            counter = 0
+            for level, type, content in ancillary:
+                assert isinstance(type,int)
+                assert isinstance(level, int)
+                levels[counter] = rffi.cast(rffi.SIGNED,level)
+                types[counter] = rffi.cast(rffi.SIGNED,type)
+                desc_per_ancillary[counter] = rffi.cast(rffi.SIGNED, (len(content)))
+                file_descr[counter] = rffi.str2charp(content, track_allocation=True)
+                counter +=1
+        else:
+            size_of_ancillary = 0
+        snd_no_msgs = rffi.cast(rffi.SIGNED, no_of_messages)
+        snd_anc_size =rffi.cast(rffi.SIGNED, size_of_ancillary)
+
+
+        bytes_sent = _c.sendmsg(self.fd, addr, addrlen, messages_length_ptr, messages_ptr, snd_no_msgs,levels,types,file_descr,desc_per_ancillary,snd_anc_size,flags)
+
+
+        if need_to_free_address:
+            address.unlock()
+        for i in range(len(messages)):
+            lltype.free(messages_ptr[i], flavor='raw', track_allocation=True)
+        lltype.free(messages_ptr, flavor='raw', track_allocation=True)
+        lltype.free(messages_length_ptr, flavor='raw', track_allocation=True)
+
+        if size_of_ancillary > 0:
+            for i in range(len(ancillary)):
+                lltype.free(file_descr[i], flavor='raw', track_allocation=True)
+        lltype.free(desc_per_ancillary, flavor='raw', track_allocation=True)
+        lltype.free(types, flavor='raw', track_allocation=True)
+        lltype.free(levels, flavor='raw', track_allocation=True)
+        lltype.free(file_descr, flavor='raw', track_allocation=True)
+
+        self.wait_for_data(True)
+        if (bytes_sent < 0) and (bytes_sent!=-1000) and (bytes_sent!=-1001) and (bytes_sent!=-1002):
+            raise last_error()
+
+        return bytes_sent
+
+
+
     def setblocking(self, block):
         if block:
             timeout = -1.0
@@ -1190,6 +1390,31 @@
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
+if _c._POSIX:
+    def CMSG_LEN( demanded_len):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param demanded_len: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is not memory alligned
+        """
+        if demanded_len < 0:
+            return 0
+        result = _c.CMSG_LEN(demanded_len)
+        return result
+
+    def CMSG_SPACE( demanded_size):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param demanded_size: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is memory alligned
+        """
+        if demanded_size < 0:
+            return 0
+        result = _c.CMSG_SPACE(demanded_size)
+        return result
+
 if _c.WIN32:
     def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -234,6 +234,12 @@
             rwin32.BOOL,
             save_err=rffi.RFFI_SAVE_LASTERROR)
 
+        CreateHardLink = external(
+            'CreateHardLink' + suffix,
+            [traits.CCHARP, traits.CCHARP, rwin32.LPSECURITY_ATTRIBUTES],
+            rwin32.BOOL,
+            save_err=rffi.RFFI_SAVE_LASTERROR)
+
     return Win32Traits
 
 def make_longlong(high, low):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -752,7 +752,8 @@
 
 # Signed, Signed *
 SIGNED = lltype.Signed
-SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
+SIGNEDP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True}))
+SIGNEDPP = lltype.Ptr(lltype.Array(SIGNEDP, hints={'nolength': True}))
 
 
 # various type mapping

From pypy.commits at gmail.com  Sun Sep  3 20:43:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 03 Sep 2017 17:43:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: cleanup
Message-ID: <59aca1ae.0caadf0a.74d70.53fc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92314:3e52029e9a5a
Date: 2017-09-04 01:42 +0100
http://bitbucket.org/pypy/pypy/changeset/3e52029e9a5a/

Log:	cleanup

diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -1,7 +1,7 @@
 import py
 
 
-class AppTestItertools: 
+class AppTestItertools(object):
     spaceconfig = dict(usemodules=['itertools'])
 
     def test_count(self):
@@ -330,11 +330,11 @@
 
     def test_chain(self):
         import itertools
-        
+
         it = itertools.chain()
         raises(StopIteration, it.next)
         raises(StopIteration, it.next)
-        
+
         it = itertools.chain([1, 2, 3])
         for x in [1, 2, 3]:
             assert it.next() == x
@@ -378,7 +378,7 @@
 
     def test_imap_wrongargs(self):
         import itertools
-        
+
         # Duplicate python 2.4 behaviour for invalid arguments
         it = itertools.imap(0, [])
         raises(StopIteration, it.next)
@@ -401,11 +401,11 @@
         for x in obj_list:
             assert it.next() == (x, )
         raises(StopIteration, it.next)
-        
+
         it = itertools.izip([1, 2, 3], [4], [5, 6])
         assert it.next() == (1, 4, 5)
         raises(StopIteration, it.next)
-        
+
         it = itertools.izip([], [], [1], [])
         raises(StopIteration, it.next)
 
@@ -423,7 +423,7 @@
 
     def test_izip_wrongargs(self):
         import itertools, re
-        
+
         # Duplicate python 2.4 behaviour for invalid arguments
         raises(TypeError, itertools.izip, None, 0)
 
@@ -442,7 +442,7 @@
 
         it = itertools.cycle([])
         raises(StopIteration, it.next)
-        
+
         it = itertools.cycle([1, 2, 3])
         for x in [1, 2, 3, 1, 2, 3, 1, 2, 3]:
             assert it.next() == x
@@ -498,7 +498,7 @@
 
     def test_tee_wrongargs(self):
         import itertools
-        
+
         raises(TypeError, itertools.tee, 0)
         raises(ValueError, itertools.tee, [], -1)
         raises(TypeError, itertools.tee, [], None)
@@ -536,7 +536,7 @@
 
     def test_groupby(self):
         import itertools
-        
+
         it = itertools.groupby([])
         raises(StopIteration, it.next)
 
@@ -613,7 +613,7 @@
             assert g.next() is x
             raises(StopIteration, g.next)
         raises(StopIteration, it.next)
-        
+
         # Grouping is based on key equality
         class AlwaysEqual(object):
             def __eq__(self, other):
@@ -647,7 +647,7 @@
 
     def test_iterables(self):
         import itertools
-    
+
         iterables = [
             itertools.chain(),
             itertools.count(),
@@ -665,7 +665,7 @@
             itertools.tee([])[0],
             itertools.tee([])[1],
             ]
-    
+
         for it in iterables:
             assert hasattr(it, '__iter__')
             assert iter(it) is it
@@ -674,7 +674,7 @@
 
     def test_docstrings(self):
         import itertools
-        
+
         assert itertools.__doc__
         methods = [
             itertools.chain,
@@ -756,15 +756,9 @@
         assert itertools.tee(a, 0) == ()
 
 
-class AppTestItertools26:
+class AppTestItertools26(object):
     spaceconfig = dict(usemodules=['itertools'])
 
-    def setup_class(cls):
-        if cls.space.is_true(cls.space.appexec([], """():
-            import sys; return sys.version_info < (2, 6)
-            """)):
-            py.test.skip("Requires Python 2.6")
-
     def test_count_overflow(self):
         import itertools, sys
         it = itertools.count(sys.maxint - 1)
@@ -1010,16 +1004,8 @@
         raises(ValueError, permutations, [1, 2], -1)
 
 
-class AppTestItertools27:
-    spaceconfig = {
-        "usemodules": ['itertools', 'struct', 'binascii'],
-    }
-
-    def setup_class(cls):
-        if cls.space.is_true(cls.space.appexec([], """():
-            import sys; return sys.version_info < (2, 7)
-            """)):
-            py.test.skip("Requires Python 2.7")
+class AppTestItertools27(object):
+    spaceconfig = {"usemodules": ['itertools', 'struct', 'binascii']}
 
     def test_compress(self):
         import itertools

From pypy.commits at gmail.com  Sun Sep  3 22:32:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 03 Sep 2017 19:32:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: itertools.islice: use same logic as
 CPython and fix #2643
Message-ID: <59acbb47.a888df0a.6a63a.98cc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92315:17c8c1d27c41
Date: 2017-09-04 03:32 +0100
http://bitbucket.org/pypy/pypy/changeset/17c8c1d27c41/

Log:	itertools.islice: use same logic as CPython and fix #2643

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -310,8 +310,7 @@
 
 islice_ignore_items_driver = jit.JitDriver(name='islice_ignore_items',
                                            greens=['tp'],
-                                           reds=['num', 'w_islice',
-                                                 'w_iterator'])
+                                           reds=['w_islice', 'w_iterator'])
 
 class W_ISlice(W_Root):
     def __init__(self, space, w_iterable, w_startstop, args_w):
@@ -325,7 +324,7 @@
             w_stop = w_startstop
         elif num_args <= 2:
             if space.is_w(w_startstop, space.w_None):
-                start = -1
+                start = 0
             else:
                 start = self.arg_int_w(w_startstop, 0,
                  "Indicies for islice() must be None or non-negative integers")
@@ -352,9 +351,10 @@
         else:
             step = 1
 
-        self.ignore = step - 1
-        self.start = start
+        self.count = 0
+        self.next = start
         self.stop = stop
+        self.step = step
 
     def arg_int_w(self, w_obj, minimum, errormsg):
         space = self.space
@@ -372,70 +372,52 @@
         return self
 
     def next_w(self):
-        if self.start >= 0:               # first call only
-            ignore = self.start
-            self.start = -1
-        else:                             # all following calls
-            ignore = self.ignore
-        stop = self.stop
-        if stop >= 0:
-            if stop <= ignore:
-                self.stop = 0   # reset the state so that a following next_w()
-                                # has no effect any more
-                if stop > 0:
-                    self._ignore_items(stop)
-                self.iterable = None
-                raise OperationError(self.space.w_StopIteration,
-                                     self.space.w_None)
-            self.stop = stop - (ignore + 1)
-        if ignore > 0:
-            self._ignore_items(ignore)
         if self.iterable is None:
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
+        self._ignore_items()
+        stop = self.stop
+        if 0 <= stop <= self.count:
+            self.iterable = None
+            raise OperationError(self.space.w_StopIteration,
+                                    self.space.w_None)
         try:
-            return self.space.next(self.iterable)
+            item = self.space.next(self.iterable)
         except OperationError as e:
             if e.match(self.space, self.space.w_StopIteration):
                 self.iterable = None
             raise
+        self.count += 1
+        oldnext = self.next
+        self.next += self.step
+        if self.next < oldnext or self.next > stop >= 0:
+            self.next = stop
+        return item
 
-    def _ignore_items(self, num):
+    def _ignore_items(self):
         w_iterator = self.iterable
-        if w_iterator is None:
-            raise OperationError(self.space.w_StopIteration, self.space.w_None)
-
         tp = self.space.type(w_iterator)
         while True:
-            islice_ignore_items_driver.jit_merge_point(tp=tp,
-                                                       num=num,
-                                                       w_islice=self,
-                                                       w_iterator=w_iterator)
+            islice_ignore_items_driver.jit_merge_point(
+                tp=tp, w_islice=self, w_iterator=w_iterator)
+            if self.count >= self.next:
+                break
             try:
                 self.space.next(w_iterator)
             except OperationError as e:
                 if e.match(self.space, self.space.w_StopIteration):
                     self.iterable = None
                 raise
-            num -= 1
-            if num <= 0:
-                break
+            self.count += 1
 
     def descr_reduce(self, space):
         if self.iterable is None:
             return space.newtuple([
                 space.type(self),
                 space.newtuple([space.iter(space.newlist([])),
-                                space.newint(0),
-                                space.newint(0),
-                                space.newint(1),
-                            ]),
+                                space.newint(0)]),
+                space.newint(0),
             ])
-        start = self.start
         stop = self.stop
-        if start == -1:
-            w_start = space.w_None
-        else:
-            w_start = space.newint(start)
         if stop == -1:
             w_stop = space.w_None
         else:
@@ -443,11 +425,15 @@
         return space.newtuple([
             space.type(self),
             space.newtuple([self.iterable,
-                            w_start,
+                            space.newint(self.next),
                             w_stop,
-                            space.newint(self.ignore + 1)]),
+                            space.newint(self.step)]),
+            space.newint(self.count),
         ])
 
+    def descr_setstate(self, space, w_state):
+        self.count = space.int_w(w_state)
+
 def W_ISlice___new__(space, w_subtype, w_iterable, w_startstop, args_w):
     r = space.allocate_instance(W_ISlice, w_subtype)
     r.__init__(space, w_iterable, w_startstop, args_w)
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -214,6 +214,9 @@
         assert list(itertools.islice(range(10), None,None)) == list(range(10))
         assert list(itertools.islice(range(10), None,None,None)) == list(range(10))
 
+        it = itertools.islice([0, 1, 2], None, None, 2)
+        assert list(it) == [0, 2]
+
         import weakref
         for args in [(1,), (None,), (0, None, 2)]:
             it = (x for x in (1, 2, 3))

From pypy.commits at gmail.com  Mon Sep  4 09:52:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 04 Sep 2017 06:52:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix merge issues
Message-ID: <59ad5a81.1b711c0a.30463.4cc3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92316:2db2c4a66195
Date: 2017-09-04 14:51 +0100
http://bitbucket.org/pypy/pypy/changeset/2db2c4a66195/

Log:	fix merge issues

diff --git a/lib-python/3/ctypes/test/test_byteswap.py b/lib-python/3/ctypes/test/test_byteswap.py
--- a/lib-python/3/ctypes/test/test_byteswap.py
+++ b/lib-python/3/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
 from binascii import hexlify
 
 from ctypes import *
-from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).decode().upper()
@@ -43,7 +42,6 @@
         with self.assertRaises(AttributeError):
             little.z = 24
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -71,7 +69,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -100,7 +97,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -129,7 +125,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -148,7 +143,6 @@
         self.assertAlmostEqual(s.value, math.pi, places=6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -176,7 +170,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -212,7 +205,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -241,7 +233,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -265,7 +256,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -72,13 +72,13 @@
                ((value >> 24) & 0xFF)
 
     def swap_8():
-        return ((value & 0x00000000000000FFL) << 56) | \
-               ((value & 0x000000000000FF00L) << 40) | \
-               ((value & 0x0000000000FF0000L) << 24) | \
-               ((value & 0x00000000FF000000L) << 8) | \
-               ((value & 0x000000FF00000000L) >> 8) | \
-               ((value & 0x0000FF0000000000L) >> 24) | \
-               ((value & 0x00FF000000000000L) >> 40) | \
+        return ((value & 0x00000000000000FF) << 56) | \
+               ((value & 0x000000000000FF00) << 40) | \
+               ((value & 0x0000000000FF0000) << 24) | \
+               ((value & 0x00000000FF000000) << 8) | \
+               ((value & 0x000000FF00000000) >> 8) | \
+               ((value & 0x0000FF0000000000) >> 24) | \
+               ((value & 0x00FF000000000000) >> 40) | \
                ((value >> 56) & 0xFF)
 
     def swap_double_float(typ):

From pypy.commits at gmail.com  Tue Sep  5 10:34:16 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 07:34:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: emit guard_nonnull_class in the short
 preamble, instead of two guards in case
Message-ID: <59aeb5e8.063a1c0a.a837f.5ef8@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92317:e66be45f3914
Date: 2017-09-05 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/e66be45f3914/

Log:	emit guard_nonnull_class in the short preamble, instead of two
	guards in case of remove_gctypeptr = True

diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -329,11 +329,14 @@
 
     def make_guards(self, op, short, optimizer):
         if self._known_class is not None:
-            short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
+                short.append(ResOperation(rop.GUARD_NONNULL, [op]))
                 short.append(ResOperation(rop.GUARD_IS_OBJECT, [op]))
-            short.append(ResOperation(rop.GUARD_CLASS,
-                                      [op, self._known_class]))
+                short.append(ResOperation(rop.GUARD_CLASS,
+                                          [op, self._known_class]))
+            else:
+                short.append(ResOperation(rop.GUARD_NONNULL_CLASS,
+                    [op, self._known_class]))
         elif self.descr is not None:
             short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7541,6 +7541,33 @@
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
+    def test_guards_before_getfields_in_short_preamble_removetypeptr(self, monkeypatch):
+        monkeypatch.setattr(self.cpu, "remove_gctypeptr", True)
+        ops = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p0)
+        """
+        expected = """
+        [p0, p1]
+        jump(p0, p1)
+        """
+        short = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
+
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]

From pypy.commits at gmail.com  Tue Sep  5 11:12:54 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 05 Sep 2017 08:12:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: allow running -A on pypy with no
 micronumpy
Message-ID: <59aebef6.51bbdf0a.3b04e.6745@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92318:4629665cda12
Date: 2017-09-05 11:47 +0300
http://bitbucket.org/pypy/pypy/changeset/4629665cda12/

Log:	allow running -A on pypy with no micronumpy

diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,7 +136,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii',
-                                   'micronumpy', 'mmap'
+                                   'mmap'
                                    ])
 
     @classmethod
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -125,52 +125,6 @@
         ten = foo.test_buffer(arr)
         assert ten == 10
 
-    @pytest.mark.skipif(only_pypy, reason='pypy only test')
-    def test_buffer_info(self):
-        try:
-            from _numpypy import multiarray as np
-        except ImportError:
-            skip('pypy built without _numpypy')
-        module = self.import_module(name='buffer_test')
-        get_buffer_info = module.get_buffer_info
-        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        arr = np.zeros((1, 10), order='F')
-        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-        assert strides[0] == 8
-        arr = np.zeros((10, 1), order='C')
-        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-        assert strides[-1] == 8
-        dt1 = np.dtype(
-             [('a', 'b'), ('b', 'i'),
-              ('sub0', np.dtype('b,i')),
-              ('sub1', np.dtype('b,i')),
-              ('sub2', np.dtype('b,i')),
-              ('sub3', np.dtype('b,i')),
-              ('sub4', np.dtype('b,i')),
-              ('sub5', np.dtype('b,i')),
-              ('sub6', np.dtype('b,i')),
-              ('sub7', np.dtype('b,i')),
-              ('c', 'i')],
-             )
-        x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
-        # pytest can catch warnings from v2.8 and up, we ship 2.5
-        import warnings
-        warnings.filterwarnings("error")
-        try:
-            try:
-                y = get_buffer_info(x, ['SIMPLE'])
-            except UserWarning as e:
-                pass
-            else:
-                assert False ,"PyPy-specific UserWarning not raised" \
-                          " on too long format string"
-        finally:
-            warnings.resetwarnings()
-        # calling get_buffer_info on x creates a memory leak,
-        # which is detected as an error at test teardown:
-        # Exception TypeError: "'NoneType' object is not callable"
-        #         in <bound method ConcreteArray.__del__ ...> ignored
-
     def test_releasebuffer(self):
         module = self.import_extension('foo', [
             ("create_test", "METH_NOARGS",
@@ -240,3 +194,55 @@
         self.debug_collect()
         assert module.get_cnt() == 0
         assert module.get_dealloc_cnt() == 1
+
+class AppTestBufferInfo(AppTestCpythonExtensionBase):
+    spaceconfig = AppTestCpythonExtensionBase.spaceconfig.copy()
+    spaceconfig['usemodules'].append('micronumpy')
+
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
+    def test_buffer_info(self):
+        try:
+            from _numpypy import multiarray as np
+        except ImportError:
+            skip('pypy built without _numpypy')
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
+        dt1 = np.dtype(
+             [('a', 'b'), ('b', 'i'),
+              ('sub0', np.dtype('b,i')),
+              ('sub1', np.dtype('b,i')),
+              ('sub2', np.dtype('b,i')),
+              ('sub3', np.dtype('b,i')),
+              ('sub4', np.dtype('b,i')),
+              ('sub5', np.dtype('b,i')),
+              ('sub6', np.dtype('b,i')),
+              ('sub7', np.dtype('b,i')),
+              ('c', 'i')],
+             )
+        x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
+        # pytest can catch warnings from v2.8 and up, we ship 2.5
+        import warnings
+        warnings.filterwarnings("error")
+        try:
+            try:
+                y = get_buffer_info(x, ['SIMPLE'])
+            except UserWarning as e:
+                pass
+            else:
+                assert False ,"PyPy-specific UserWarning not raised" \
+                          " on too long format string"
+        finally:
+            warnings.resetwarnings()
+        # calling get_buffer_info on x creates a memory leak,
+        # which is detected as an error at test teardown:
+        # Exception TypeError: "'NoneType' object is not callable"
+        #         in <bound method ConcreteArray.__del__ ...> ignored
+
+
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -26,6 +26,8 @@
 NULL = lltype.nullptr(rffi.VOIDP.TO)
 
 class TestNDArrayObject(BaseApiTest):
+    spaceconfig = AppTestCpythonExtensionBase.spaceconfig.copy()
+    spaceconfig['usemodules'].append('micronumpy')
 
     def test_Check(self, space, api):
         a = array(space, [10, 5, 3])

From pypy.commits at gmail.com  Tue Sep  5 11:29:34 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 05 Sep 2017 08:29:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <59aec2de.c59c1c0a.ab643.685a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92319:2bf77ef25a29
Date: 2017-09-05 18:28 +0300
http://bitbucket.org/pypy/pypy/changeset/2bf77ef25a29/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -73,3 +73,7 @@
 
 Add support for leakfinder in cpyext tests (disabled for now, due to too many
 failures).
+
+.. branch: pypy_swappedbytes
+
+Added ``_swappedbytes_`` support for ``ctypes.Structure``

From pypy.commits at gmail.com  Tue Sep  5 15:44:43 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:43 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: support for more operations,
 including shifts
Message-ID: <59aefeab.64a8df0a.a5400.e779@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92320:33863bdafaae
Date: 2017-09-03 09:26 +0200
http://bitbucket.org/pypy/pypy/changeset/33863bdafaae/

Log:	support for more operations, including shifts

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -50,13 +50,13 @@
     def _consider_binop_symm(self, op, position):
         self._consider_binop_part(op, position, symm=True)
 
-    #consider_int_mul = _consider_binop_symm
-    #consider_int_and = _consider_binop_symm
-    #consider_int_or  = _consider_binop_symm
-    #consider_int_xor = _consider_binop_symm
+    consider_int_mul = _consider_binop_symm
+    consider_int_and = _consider_binop_symm
+    consider_int_or  = _consider_binop_symm
+    consider_int_xor = _consider_binop_symm
 
-    #consider_int_mul_ovf = _consider_binop_symm
-    #consider_int_sub_ovf = _consider_binop
+    consider_int_mul_ovf = _consider_binop_symm
+    consider_int_sub_ovf = _consider_binop
     consider_int_add_ovf = _consider_binop_symm
 
     def _consider_lea(self, op, loc):
@@ -73,12 +73,15 @@
 
     consider_nursery_ptr_increment = consider_int_add
 
-    def Xconsider_int_lshift(self, op, position):
-        if not isinstance(op.getarg(1), Const):
+    def consider_int_lshift(self, op, position):
+        x, y = op.getarg(0), op.getarg(1)
+        if not isinstance(y, Const):
             self.longevity.fixed_register(position, ecx, op.getarg(1))
+        if not isinstance(x, Const):
+            self.longevity.try_use_same_register(x, op)
 
-    #consider_int_rshift  = consider_int_lshift
-    #consider_uint_rshift = consider_int_lshift
+    consider_int_rshift  = consider_int_lshift
+    consider_uint_rshift = consider_int_lshift
 
     def Xconsider_call_malloc_nursery(self, op, position):
         self.longevity.fixed_register(position, ecx, op)
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -102,6 +102,35 @@
         # coalescing makes sure that i0 (and thus i71) lands in edi
         assert len([entry for entry in self.log if entry.name == "mov"]) == 2
 
+    def test_coalescing_mul(self):
+        # won't test all operations, but at least check a second one
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_mul(i0, i1)
+        i8 = int_mul(i7, i3)
+        i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) []
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 2
+
+    def test_lshift(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i5 = int_add(i2, i3)
+        i7 = int_lshift(i0, i5)
+        i8 = int_lshift(i7, i3)
+        i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) []
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        # 3 moves for arguments, 1 move for result
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 4
+
     def test_binop_dont_swap_unnecessarily(self):
         ops = '''
         [i0, i1, i2, i3]

From pypy.commits at gmail.com  Tue Sep  5 15:44:47 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:47 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: test for later
Message-ID: <59aefeaf.64a8df0a.a5400.e781@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92322:96fe37156480
Date: 2017-09-03 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/96fe37156480/

Log:	test for later

diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -88,6 +88,21 @@
         # i0 and i1, one for the result to the stack
         assert len([entry for entry in self.log if entry.name == "mov"]) == 3
 
+    @pytest.mark.skip("later")
+    def test_same_stack_entry_many_times(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1)
+        i8 = int_add(i2, i1)
+        i9 = int_add(i3, i1)
+        i10 = int_is_true(i9)
+        guard_true(i10) [i8]
+        finish(i7)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        # 4 moves for arguments, 1 for result
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 5
+
     def test_coalescing(self):
         ops = '''
         [i0, i1, i2, i3]

From pypy.commits at gmail.com  Tue Sep  5 15:44:49 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:49 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: fix a bug with consta
 arguments
Message-ID: <59aefeb1.5d87df0a.879f7.bddb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92323:494789502df8
Date: 2017-09-03 11:55 +0200
http://bitbucket.org/pypy/pypy/changeset/494789502df8/

Log:	fix a bug with consta arguments

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -512,11 +512,12 @@
         # and if x lives longer than the current operation while y dies, then
         # swap the role of 'x' and 'y'
         if (symm and not isinstance(xloc, RegLoc) and
-                isinstance(argloc, RegLoc) and
-                self.rm.longevity[x].last_usage > self.rm.position and
-                self.longevity[y].last_usage == self.rm.position):
-            x, y = y, x
-            argloc = self.loc(y)
+                isinstance(argloc, RegLoc)):
+            if ((x not in self.rm.longevity or
+                    self.rm.longevity[x].last_usage > self.rm.position) and
+                    self.rm.longevity[y].last_usage == self.rm.position):
+                x, y = y, x
+                argloc = self.loc(y)
         #
         args = op.getarglist()
         loc = self.rm.force_result_in_reg(op, x, args)
@@ -613,7 +614,6 @@
         vx = op.getarg(0)
         vy = op.getarg(1)
         arglocs = [self.loc(vx), self.loc(vy)]
-        args = op.getarglist()
         if (vx in self.rm.reg_bindings or vy in self.rm.reg_bindings or
             isinstance(vx, Const) or isinstance(vy, Const)):
             pass
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -38,8 +38,10 @@
 
         # For symmetrical operations, if y won't be used after the current
         # operation finishes, but x will be, then swap the role of 'x' and 'y'
-        if (self.longevity[x].last_usage > position and
-                self.longevity[y].last_usage == position):
+        if (symm and isinstance(x, ConstInt) or (
+                not isinstance(y, ConstInt) and
+                self.longevity[x].last_usage > position and
+                self.longevity[y].last_usage == position)):
             x, y = y, x
         self.longevity.try_use_same_register(x, op)
 
@@ -77,6 +79,12 @@
     consider_int_rshift  = consider_int_lshift
     consider_uint_rshift = consider_int_lshift
 
+    def consider_uint_mul_high(self, op, position):
+        # could do a lot more, but I suspect not worth it
+        # just block eax and edx
+        self.longevity.fixed_register(position, eax)
+        self.longevity.fixed_register(position, edx)
+
     def Xconsider_call_malloc_nursery(self, op, position):
         self.longevity.fixed_register(position, ecx, op)
         self.longevity.fixed_register(position, edx)
@@ -108,7 +116,8 @@
         assert effectinfo is not None
         oopspecindex = effectinfo.oopspecindex
         if oopspecindex != EffectInfo.OS_NONE:
-            raise NotImplementedError
+            # XXX safe default: do nothing
+            return
         self._consider_call(op, position)
 
     consider_call_i = _consider_real_call
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -72,6 +72,15 @@
         self.interpret(ops, [5, 6, 7, 8])
         assert len([entry for entry in self.log if entry.args[0] == "int_add"]) == 1
 
+    def test_bug_const(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i9 = int_add(1, i3)
+        finish(i9)
+        '''
+        # does not crash
+        self.interpret(ops, [5, 6, 7, 8])
+        assert len([entry for entry in self.log if entry.args[0] == "int_add"]) == 1
 
     def test_call_use_correct_regs(self):
         ops = '''
@@ -161,5 +170,5 @@
         op = self.log[5]
         assert op.name == "op"
         # make sure that the arguments of the third op are not swapped (since
-        # that would break coalescing between i7 and i9
+        # that would break coalescing between i7 and i9)
         assert op.args[1][0] is add1.args[-1]

From pypy.commits at gmail.com  Tue Sep  5 15:44:45 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:45 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: cleanup
Message-ID: <59aefead.51bbdf0a.3b04e.c56a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92321:5d156f96018b
Date: 2017-09-03 09:28 +0200
http://bitbucket.org/pypy/pypy/changeset/5d156f96018b/

Log:	cleanup

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -36,9 +36,8 @@
         x = op.getarg(0)
         y = op.getarg(1)
 
-        # For symmetrical operations, if 'y' is already in a register
-        # and won't be used after the current operation finishes,
-        # then swap the role of 'x' and 'y'
+        # For symmetrical operations, if y won't be used after the current
+        # operation finishes, but x will be, then swap the role of 'x' and 'y'
         if (self.longevity[x].last_usage > position and
                 self.longevity[y].last_usage == position):
             x, y = y, x
@@ -59,11 +58,6 @@
     consider_int_sub_ovf = _consider_binop
     consider_int_add_ovf = _consider_binop_symm
 
-    def _consider_lea(self, op, loc):
-        argloc = self.loc(op.getarg(1))
-        resloc = self.force_allocate_reg(op)
-        self.perform(op, [loc, argloc], resloc)
-
     def consider_int_add(self, op, position):
         y = op.getarg(1)
         if isinstance(y, ConstInt) and rx86.fits_in_32bits(y.value):
@@ -76,7 +70,7 @@
     def consider_int_lshift(self, op, position):
         x, y = op.getarg(0), op.getarg(1)
         if not isinstance(y, Const):
-            self.longevity.fixed_register(position, ecx, op.getarg(1))
+            self.longevity.fixed_register(position, ecx, y)
         if not isinstance(x, Const):
             self.longevity.try_use_same_register(x, op)
 

From pypy.commits at gmail.com  Tue Sep  5 15:44:51 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:51 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: fix bug for a call argument
 appearing twice
Message-ID: <59aefeb3.c80d1c0a.5e1cd.980a@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92324:925ba08f5490
Date: 2017-09-03 12:05 +0200
http://bitbucket.org/pypy/pypy/changeset/925ba08f5490/

Log:	fix bug for a call argument appearing twice

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -167,24 +167,28 @@
     def hint(self, position, args, argtypes, save_all_regs):
         hinted_xmm = []
         hinted_gpr = []
+        hinted_args = []
         for i in range(len(args)):
             arg = args[i]
             if arg.type == "f":
                 tgt = self._unused_xmm()
-                if tgt is not None and not arg.is_constant():
+                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
                     self.longevity.fixed_register(position, tgt, arg)
                     hinted_xmm.append(tgt)
+                    hinted_args.append(arg)
             elif i < len(argtypes) and argtypes[i] == 'S':
                 # Singlefloat argument
                 tgt = self._unused_xmm()
-                if tgt is not None and not arg.is_constant():
+                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
                     self.longevity.fixed_register(position, tgt, arg)
                     hinted_xmm.append(tgt)
+                    hinted_args.append(arg)
             else:
                 tgt = self._unused_gpr()
-                if tgt is not None and not arg.is_constant():
+                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
                     self.longevity.fixed_register(position, tgt, arg)
                     hinted_gpr.append(tgt)
+                    hinted_args.append(arg)
         # block all remaining registers that are not caller save
         # XXX the case save_all_regs == 1 (save callee-save regs + gc ptrs) is
         # no expressible atm
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -97,6 +97,22 @@
         # i0 and i1, one for the result to the stack
         assert len([entry for entry in self.log if entry.name == "mov"]) == 3
 
+    def test_call_use_argument_twice(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i7 = int_add(i0, i1)
+        i8 = int_add(i2, 13)
+        i9 = call_i(ConstClass(f2ptr), i7, i7, descr=f2_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) [i8]
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        # two moves are needed from the stack frame to registers for arguments
+        # i0 and i1, one for the result to the stack
+        # one for the copy to the other argument register
+        assert len([entry for entry in self.log if entry.name == "mov"]) == 4
+
     @pytest.mark.skip("later")
     def test_same_stack_entry_many_times(self):
         ops = '''

From pypy.commits at gmail.com  Tue Sep  5 15:44:53 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:53 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: convenience method in tests
Message-ID: <59aefeb5.1986df0a.9c758.d53e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92325:a491c78e08f8
Date: 2017-09-04 09:27 +0200
http://bitbucket.org/pypy/pypy/changeset/a491c78e08f8/

Log:	convenience method in tests

diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -61,6 +61,9 @@
         for l in self.log:
             print l
 
+    def filter_log_moves(self):
+        return [entry for entry in self.log if entry.name == "mov"]
+
     def test_unused(self):
         ops = '''
         [i0, i1, i2, i3]
@@ -95,7 +98,7 @@
         self.interpret(ops, [5, 6, 7, 8])
         # two moves are needed from the stack frame to registers for arguments
         # i0 and i1, one for the result to the stack
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 3
+        assert len(self.filter_log_moves()) == 3
 
     def test_call_use_argument_twice(self):
         ops = '''
@@ -111,7 +114,7 @@
         # two moves are needed from the stack frame to registers for arguments
         # i0 and i1, one for the result to the stack
         # one for the copy to the other argument register
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 4
+        assert len(self.filter_log_moves()) == 4
 
     @pytest.mark.skip("later")
     def test_same_stack_entry_many_times(self):
@@ -126,7 +129,7 @@
         '''
         self.interpret(ops, [5, 6, 7, 8])
         # 4 moves for arguments, 1 for result
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 5
+        assert len(self.filter_log_moves()) == 5
 
     def test_coalescing(self):
         ops = '''
@@ -140,7 +143,7 @@
         '''
         self.interpret(ops, [5, 6, 7, 8])
         # coalescing makes sure that i0 (and thus i71) lands in edi
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 2
+        assert len(self.filter_log_moves()) == 2
 
     def test_coalescing_mul(self):
         # won't test all operations, but at least check a second one
@@ -154,7 +157,7 @@
         finish(i9)
         '''
         self.interpret(ops, [5, 6, 7, 8])
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 2
+        assert len(self.filter_log_moves()) == 2
 
     def test_lshift(self):
         ops = '''
@@ -169,7 +172,7 @@
         '''
         self.interpret(ops, [5, 6, 7, 8])
         # 3 moves for arguments, 1 move for result
-        assert len([entry for entry in self.log if entry.name == "mov"]) == 4
+        assert len(self.filter_log_moves()) == 4
 
     def test_binop_dont_swap_unnecessarily(self):
         ops = '''

From pypy.commits at gmail.com  Tue Sep  5 15:44:55 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:55 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: use LEA even for additions
 that involve the stack
Message-ID: <59aefeb7.8a841c0a.81513.8979@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92326:ec41cc35c5d8
Date: 2017-09-04 09:30 +0200
http://bitbucket.org/pypy/pypy/changeset/ec41cc35c5d8/

Log:	use LEA even for additions that involve the stack (that way we can
	move the stack argument to a register, where it might be used soon
	again)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -531,28 +531,25 @@
         loc, argloc = self._consider_binop_part(op, symm=True)
         self.perform(op, [loc, argloc], loc)
 
-    def _consider_lea(self, op, loc):
+    def _consider_lea(self, op):
+        loc = self.make_sure_var_in_reg(op.getarg(0))
         argloc = self.loc(op.getarg(1))
         resloc = self.force_allocate_reg(op)
         self.perform(op, [loc, argloc], resloc)
 
     def consider_int_add(self, op):
-        loc = self.loc(op.getarg(0))
         y = op.getarg(1)
-        if (isinstance(loc, RegLoc) and
-            isinstance(y, ConstInt) and rx86.fits_in_32bits(y.value)):
-            self._consider_lea(op, loc)
+        if isinstance(y, ConstInt) and rx86.fits_in_32bits(y.value):
+            self._consider_lea(op)
         else:
             self._consider_binop_symm(op)
 
     consider_nursery_ptr_increment = consider_int_add
 
     def consider_int_sub(self, op):
-        loc = self.loc(op.getarg(0))
         y = op.getarg(1)
-        if (isinstance(loc, RegLoc) and
-            isinstance(y, ConstInt) and rx86.fits_in_32bits(-y.value)):
-            self._consider_lea(op, loc)
+        if isinstance(y, ConstInt) and rx86.fits_in_32bits(-y.value):
+            self._consider_lea(op)
         else:
             self._consider_binop(op)
 
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -85,6 +85,17 @@
         self.interpret(ops, [5, 6, 7, 8])
         assert len([entry for entry in self.log if entry.args[0] == "int_add"]) == 1
 
+    def test_use_lea_even_for_stack(self):
+        ops = '''
+        [i0, i1, i2, i3]
+        i9 = int_add(i3, 16)
+        i4 = int_add(i3, 26)
+        i6 = int_add(i9, i4)
+        finish(i6)
+        '''
+        self.interpret(ops, [5, 6, 7, 8])
+        assert len(self.filter_log_moves()) == 2
+
     def test_call_use_correct_regs(self):
         ops = '''
         [i0, i1, i2, i3]

From pypy.commits at gmail.com  Tue Sep  5 15:44:57 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:57 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: hints for
 call_malloc_nursery
Message-ID: <59aefeb9.6285df0a.6011.6ccb@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92327:c5f253b5d0c1
Date: 2017-09-04 09:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c5f253b5d0c1/

Log:	hints for call_malloc_nursery

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -85,7 +85,7 @@
         self.longevity.fixed_register(position, eax)
         self.longevity.fixed_register(position, edx)
 
-    def Xconsider_call_malloc_nursery(self, op, position):
+    def consider_call_malloc_nursery(self, op, position):
         self.longevity.fixed_register(position, ecx, op)
         self.longevity.fixed_register(position, edx)
 
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -48,6 +48,9 @@
         return Assembler386.regalloc_perform_guard(self, guard_op, faillocs,
                 arglocs, resloc, frame_depth)
 
+    def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap):
+        self._log("malloc_cond", size) # always uses edx and ecx
+
 
 class TestCheckRegistersExplicitly(test_regalloc_integration.BaseTestRegalloc):
     def setup_class(cls):
@@ -202,3 +205,24 @@
         # make sure that the arguments of the third op are not swapped (since
         # that would break coalescing between i7 and i9)
         assert op.args[1][0] is add1.args[-1]
+
+    def test_malloc(self, monkeypatch):
+        ops = '''
+        [i0]
+        label(i0, descr=targettoken)
+        i1 = int_add(i0, 1) # this is using ecx or edx because it fits
+        i6 = int_add(i0, 6) # this is using ecx or edx because it fits
+        i2 = int_add(i6, i1)
+        p0 = call_malloc_nursery(16)
+        gc_store(p0, 0, 83944, 8)
+        gc_store(p0, 8, i2, 8)
+        i10 = int_is_true(i2)
+        guard_true(i10) [p0, i0]
+        finish(p0)
+        '''
+        monkeypatch.setattr(self.cpu.gc_ll_descr, "get_nursery_top_addr", lambda: 61)
+        monkeypatch.setattr(self.cpu.gc_ll_descr, "get_nursery_free_addr", lambda: 68)
+        self.interpret(ops, [0], run=False)
+        # 2 moves, because the call_malloc_nursery hints prevent using ecx and
+        # edx for any of the integer results
+        assert len(self.filter_log_moves()) == 2

From pypy.commits at gmail.com  Tue Sep  5 15:44:58 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:44:58 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: extract a series of int ops
 from a real trace (and implement int_neg)
Message-ID: <59aefeba.8fb6df0a.baa99.7241@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92328:d2f37627db2e
Date: 2017-09-05 18:33 +0200
http://bitbucket.org/pypy/pypy/changeset/d2f37627db2e/

Log:	extract a series of int ops from a real trace (and implement
	int_neg)

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1000,6 +1000,7 @@
     consider_cond_call_value_r = consider_cond_call
 
     def consider_call_malloc_nursery(self, op):
+        # YYY what's the reason for using a fixed register for the result?
         size_box = op.getarg(0)
         assert isinstance(size_box, ConstInt)
         size = size_box.getint()
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -32,18 +32,27 @@
         # no hint by default
         pass
 
+
+    def consider_int_neg(self, op, position):
+        self.longevity.try_use_same_register(op.getarg(0), op)
+    consider_int_invert = consider_int_neg
+
     def _consider_binop_part(self, op, position, symm=False):
         x = op.getarg(0)
         y = op.getarg(1)
 
         # For symmetrical operations, if y won't be used after the current
         # operation finishes, but x will be, then swap the role of 'x' and 'y'
-        if (symm and isinstance(x, ConstInt) or (
-                not isinstance(y, ConstInt) and
-                self.longevity[x].last_usage > position and
-                self.longevity[y].last_usage == position)):
-            x, y = y, x
-        self.longevity.try_use_same_register(x, op)
+        if symm:
+            if isinstance(x, Const):
+                x, y = y, x
+            elif (not isinstance(y, Const) and
+                    self.longevity[x].last_usage > position and
+                    self.longevity[y].last_usage == position):
+                x, y = y, x
+
+        if not isinstance(x, Const):
+            self.longevity.try_use_same_register(x, op)
 
     def _consider_binop(self, op, position):
         self._consider_binop_part(op, position)
@@ -89,8 +98,8 @@
         self.longevity.fixed_register(position, ecx, op)
         self.longevity.fixed_register(position, edx)
 
-    #consider_call_malloc_nursery_varsize = consider_call_malloc_nursery
-    #consider_call_malloc_nursery_varsize_frame = consider_call_malloc_nursery
+    consider_call_malloc_nursery_varsize = consider_call_malloc_nursery
+    consider_call_malloc_nursery_varsize_frame = consider_call_malloc_nursery
 
 
     def _call(self, op, position, args, save_all_regs=False):
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -226,3 +226,36 @@
         # 2 moves, because the call_malloc_nursery hints prevent using ecx and
         # edx for any of the integer results
         assert len(self.filter_log_moves()) == 2
+
+    def test_flowcontext(self):
+        # real index manipulation for a slicing operation done when translating
+        # on top of pypy
+        ops = """
+        [i1, i2]
+        i3 = int_and(i1, 255)
+        i4 = int_rshift(i1, 8)
+        i5 = int_and(i4, 255)
+        i6 = int_lt(0, i5)
+        guard_false(i6) [i1]
+        i7 = int_eq(i3, 0)
+        guard_false(i7) [i1]
+        i8 = int_neg(i3)
+        i9 = int_lt(i8, 0)
+        guard_true(i9) [i1]
+        i10 = int_lt(i2, 0)
+        guard_false(i10) [i1]
+        i11 = int_add(i8, i2)
+        i12 = int_lt(i11, 0)
+        guard_false(i12) [i1]
+        i13 = int_gt(i11, i2)
+        guard_false(i13) [i1]
+        i14 = int_sub(i2, i11)
+        i15 = int_is_zero(i14)
+        guard_false(i15) [i1]
+        # this simulates the arraycopy call
+        i16 = call_i(ConstClass(f2ptr), i11, i14, descr=f2_calldescr)
+        finish(i16)
+        """
+        self.interpret(ops, [0], run=False)
+        # 4 moves, three for args, one for result
+        assert len(self.filter_log_moves()) == 4

From pypy.commits at gmail.com  Tue Sep  5 15:50:20 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 12:50:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: in the fast zip(intlist1,
 intlist2) implementation, don't wrap and unwrap all
Message-ID: <59aefffc.9598df0a.6c3e9.9fba@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92329:ffbad4ff0842
Date: 2017-09-05 21:45 +0200
http://bitbucket.org/pypy/pypy/changeset/ffbad4ff0842/

Log:	in the fast zip(intlist1, intlist2) implementation, don't wrap and
	unwrap all the ints

diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -31,23 +31,23 @@
     class cls(W_AbstractTupleObject):
         _immutable_fields_ = ['value%s' % i for i in iter_n]
 
-        def __init__(self, space, *values_w):
+        def __init__(self, space, *values):
             self.space = space
-            assert len(values_w) == typelen
+            assert len(values) == typelen
             for i in iter_n:
-                w_obj = values_w[i]
+                obj = values[i]
                 val_type = typetuple[i]
                 if val_type == int:
-                    unwrapped = w_obj.int_w(space)
+                    assert isinstance(obj, int)
                 elif val_type == float:
-                    unwrapped = w_obj.float_w(space)
+                    assert isinstance(obj, float)
                 elif val_type == str:
-                    unwrapped = w_obj.str_w(space)
+                    assert isinstance(obj, str)
                 elif val_type == object:
-                    unwrapped = w_obj
+                    pass
                 else:
                     raise AssertionError
-                setattr(self, 'value%s' % i, unwrapped)
+                setattr(self, 'value%s' % i, obj)
 
         def length(self):
             return typelen
@@ -150,10 +150,10 @@
         w_arg1, w_arg2 = list_w
         if type(w_arg1) is W_IntObject:
             if type(w_arg2) is W_IntObject:
-                return Cls_ii(space, w_arg1, w_arg2)
+                return Cls_ii(space, space.int_w(w_arg1), space.int_w(w_arg2))
         elif type(w_arg1) is W_FloatObject:
             if type(w_arg2) is W_FloatObject:
-                return Cls_ff(space, w_arg1, w_arg2)
+                return Cls_ff(space, space.float_w(w_arg1), space.float_w(w_arg2))
         return Cls_oo(space, w_arg1, w_arg2)
     else:
         raise NotSpecialised
@@ -170,10 +170,9 @@
 # faster to move the decision out of the loop.
 
 @specialize.arg(1)
-def _build_zipped_spec(space, Cls, lst1, lst2, wrap1, wrap2):
+def _build_zipped_spec(space, Cls, lst1, lst2):
     length = min(len(lst1), len(lst2))
-    return [Cls(space, wrap1(lst1[i]),
-                       wrap2(lst2[i])) for i in range(length)]
+    return [Cls(space, lst1[i], lst2[i]) for i in range(length)]
 
 def _build_zipped_spec_oo(space, w_list1, w_list2):
     strat1 = w_list1.strategy
@@ -200,8 +199,7 @@
             intlist2 = w_list2.getitems_int()
             if intlist2 is not None:
                 lst_w = _build_zipped_spec(
-                        space, Cls_ii, intlist1, intlist2,
-                        space.newint, space.newint)
+                        space, Cls_ii, intlist1, intlist2)
                 return space.newlist(lst_w)
         else:
             floatlist1 = w_list1.getitems_float()
@@ -209,8 +207,7 @@
                 floatlist2 = w_list2.getitems_float()
                 if floatlist2 is not None:
                     lst_w = _build_zipped_spec(
-                        space, Cls_ff, floatlist1, floatlist2, space.newfloat,
-                        space.newfloat)
+                        space, Cls_ff, floatlist1, floatlist2)
                     return space.newlist(lst_w)
 
         lst_w = _build_zipped_spec_oo(space, w_list1, w_list2)

From pypy.commits at gmail.com  Wed Sep  6 02:18:26 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Tue, 05 Sep 2017 23:18:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: make sure that
 get_strategy_from_list_objects does not have random effects :-(
Message-ID: <59af9332.1986df0a.9c758.a698@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92330:2769e258135f
Date: 2017-09-06 08:17 +0200
http://bitbucket.org/pypy/pypy/changeset/2769e258135f/

Log:	make sure that get_strategy_from_list_objects does not have random
	effects :-(

diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -116,10 +116,10 @@
     if check_int_or_float:
         for w_obj in list_w:
             if type(w_obj) is W_IntObject:
-                if longlong2float.can_encode_int32(space.int_w(w_obj)):
+                if longlong2float.can_encode_int32(w_obj.int_w(space)):
                     continue    # ok
             elif type(w_obj) is W_FloatObject:
-                if longlong2float.can_encode_float(space.float_w(w_obj)):
+                if longlong2float.can_encode_float(w_obj.float_w(space)):
                     continue    # ok
             break
         else:

From pypy.commits at gmail.com  Wed Sep  6 03:01:16 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 00:01:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve error message: include name of
 calling graph, not just op
Message-ID: <59af9d3c.ccb2df0a.2733d.d0c5@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92331:e2454a241194
Date: 2017-09-06 09:00 +0200
http://bitbucket.org/pypy/pypy/changeset/e2454a241194/

Log:	improve error message: include name of calling graph, not just op

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -186,7 +186,8 @@
         return (fnaddr, calldescr)
 
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None,
+                     calling_graph=None):
         """Return the calldescr that describes all calls done by 'op'.
         This returns a calldescr that we can put in the corresponding
         call operation in the calling jitcode.  It gets an effectinfo
@@ -202,13 +203,13 @@
         ARGS = FUNC.ARGS
         if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
+                "operation %r in %s: calling a function with signature %r, "
                 "but passing actual arguments (ignoring voids) of types %r"
-                % (op, FUNC, NON_VOID_ARGS))
+                % (op, calling_graph, FUNC, NON_VOID_ARGS))
         if RESULT != FUNC.RESULT:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
-                "but the actual return type is %r" % (op, FUNC, RESULT))
+                "%r in %s: calling a function with signature %r, "
+                "but the actual return type is %r" % (op, calling_graph, FUNC, RESULT))
         # ok
         # get the 'elidable' and 'loopinvariant' flags from the function object
         elidable = False
@@ -217,7 +218,7 @@
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
-                "%r: getcalldescr() with a non-default call ABI" % (op,))
+                "%r in %s: getcalldescr() with a non-default call ABI" % (op, calling_graph))
             func = getattr(funcobj, '_callable', None)
             elidable = getattr(func, "_elidable_function_", False)
             loopinvariant = getattr(func, "_jit_loop_invariant_", False)
@@ -245,11 +246,11 @@
                 if not error:
                     continue
                 raise Exception(
-                    "%r is an indirect call to a family of functions "
+                    "%r in %s is an indirect call to a family of functions "
                     "(or methods) that includes %r. However, the latter "
                     "is marked %r. You need to use an indirection: replace "
                     "it with a non-marked function/method which calls the "
-                    "marked function." % (op, graph, error))
+                    "marked function." % (op, calling_graph, graph, error))
         # build the extraeffect
         random_effects = self.randomeffects_analyzer.analyze(op)
         if random_effects:
@@ -278,21 +279,21 @@
         if loopinvariant:
             if extraeffect != EffectInfo.EF_LOOPINVARIANT:
                 raise Exception(
-                "in operation %r: this calls a _jit_loop_invariant_ function,"
+                "operation %r in %s: this calls a _jit_loop_invariant_ function,"
                 " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+                " effects): EF=%s" % (op, calling_graph, extraeffect))
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
                                    EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
                 raise Exception(
-                "in operation %r: this calls an elidable function,"
+                "operation %r in %s: this calls an elidable function,"
                 " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+                " effects): EF=%s" % (op, calling_graph, extraeffect))
             elif RESULT is lltype.Void:
                 raise Exception(
-                    "in operation %r: this calls an elidable function "
-                    "but the function has no result" % (op, ))
+                    "operation %r in %s: this calls an elidable function "
+                    "but the function has no result" % (op, calling_graph))
         #
         effectinfo = effectinfo_from_writeanalyze(
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -64,6 +64,7 @@
         self.cpu = cpu
         self.callcontrol = callcontrol
         self.portal_jd = portal_jd   # non-None only for the portal graph(s)
+        self.graph = None
 
     def transform(self, graph):
         self.graph = graph
@@ -424,7 +425,8 @@
         of 'residual_call_xxx' are the function to call, and its calldescr."""
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex=oopspecindex,
                                                   extraeffect=extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         op1 = self.rewrite_call(op, 'residual_call',
                                 [op.args[0]] + extraargs, calldescr=calldescr)
         if may_call_jitcodes or self.callcontrol.calldescr_canraise(calldescr):
@@ -1613,7 +1615,9 @@
         if len(op.args) > 4 + 2 or have_floats:
             raise Exception("Conditional call does not support floats or more than 4 arguments")
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
-        calldescr = self.callcontrol.getcalldescr(callop)
+        calldescr = self.callcontrol.getcalldescr(
+                callop,
+                calling_graph=self.graph)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
@@ -1924,7 +1928,8 @@
                              extradescr=None):
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
                                                   extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         if extraeffect is not None:
             assert (is_test_calldescr(calldescr)      # for tests
                     or calldescr.get_extra_info().extraeffect == extraeffect)
@@ -1954,7 +1959,8 @@
                             [c_func] + [varoftype(T) for T in argtypes],
                             varoftype(resulttype))
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
-                                                  effectinfo)
+                                                  effectinfo,
+                                                  calling_graph=self.graph)
         if isinstance(c_func.value, str):    # in tests only
             func = c_func.value
         else:
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -329,19 +329,19 @@
     @jit.elidable
     def f1(n, m):
         l.append(n)
-    def f(n, m):
+    def fancy_graph_name(n, m):
         f1(n, m)
         return n + m
 
-    rtyper = support.annotate(f, [7, 9])
+    rtyper = support.annotate(fancy_graph_name, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
     cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
     res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
+    [f_graph] = [x for x in res if x.func is fancy_graph_name]
     call_op = f_graph.startblock.operations[0]
     assert call_op.opname == 'direct_call'
-    with py.test.raises(Exception):
-        call_descr = cc.getcalldescr(call_op)
+    x = py.test.raises(Exception, cc.getcalldescr, call_op, calling_graph=f_graph)
+    assert "fancy_graph_name" in str(x.value)
 
 def test_can_or_cannot_collect():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -73,7 +73,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None, calling_graph=None):
         try:
             name = op.args[0].value._obj._name
             if 'cannot_raise' in name or name.startswith('cast_'):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -48,7 +48,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         return 'calldescr'
     def calldescr_canraise(self, calldescr):
         return True
@@ -106,7 +106,7 @@
     def guess_call_kind(self, op):
         return 'builtin'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         assert oopspecindex is not None    # in this test
         EI = effectinfo.EffectInfo
         if oopspecindex != EI.OS_ARRAYCOPY:
diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py
--- a/rpython/jit/codewriter/test/test_list.py
+++ b/rpython/jit/codewriter/test/test_list.py
@@ -39,7 +39,7 @@
 class FakeCallControl:
     class getcalldescr(AbstractDescr):
         def __init__(self, op, oopspecindex=0, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
             self.op = op
             self.oopspecindex = oopspecindex
         def __repr__(self):

From pypy.commits at gmail.com  Wed Sep  6 03:54:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 06 Sep 2017 00:54:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: checks that specialize:argtype() makes
 two copies of a
Message-ID: <59afa9c1.938edf0a.78252.a5c0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92332:5bcb6691636b
Date: 2017-09-06 09:54 +0200
http://bitbucket.org/pypy/pypy/changeset/5bcb6691636b/

Log:	checks that specialize:argtype() makes two copies of a function f(),
	one for the base class and one for the subclass

diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4644,6 +4644,25 @@
         s_exc = a.binding(graphof(a, f).exceptblock.inputargs[1])
         assert not s_exc.can_be_none()
 
+    def test_specialize_argtype_with_subclasses(self):
+        # checks that specialize:argtype() makes two copies of a
+        # function f(), one for the base class and one for the subclass
+        class A:
+            def foo(self):
+                return 123
+        class B(A):
+            def foo(self):
+                return 456
+        def f(x):
+            return x.foo()
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            if y > 5:
+                f(A())
+            return f(B())
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]).const == 456
+
 
 def g(n):
     return [0, 1, 2, n]

From pypy.commits at gmail.com  Wed Sep  6 04:41:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 06 Sep 2017 01:41:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: translation fix: avoids calling the
 general methods like W_Root.int_w() on objects that are known to be e.g.
 W_IntObject. This should make 2769e258135f unnecessary.
Message-ID: <59afb4c6.c40c1c0a.4e2ab.7bde@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92333:89a8f8f1a1e2
Date: 2017-09-06 09:40 +0100
http://bitbucket.org/pypy/pypy/changeset/89a8f8f1a1e2/

Log:	translation fix: avoids calling the general methods like
	W_Root.int_w() on objects that are known to be e.g. W_IntObject.
	This should make 2769e258135f unnecessary.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1629,12 +1629,15 @@
         # return text_w(w_obj) or None
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    @specialize.argtype(1)
     def bytes_w(self, w_obj):
         """ Takes an application level :py:class:`bytes`
             (on PyPy2 this equals `str`) and returns a rpython byte string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
+    @specialize.argtype(1)
     def text_w(self, w_obj):
         """ PyPy2 takes either a :py:class:`str` and returns a
             rpython byte string, or it takes an :py:class:`unicode`
@@ -1644,6 +1647,7 @@
             On PyPy3 it takes a :py:class:`str` and it will return
             an utf-8 encoded rpython string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1692,6 +1696,7 @@
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
         return chr(value)
 
+    @specialize.argtype(1)
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1704,26 +1709,35 @@
         If allow_conversion=False, w_obj needs to be an app-level int or a
         subclass.
         """
+        assert w_obj is not None
         return w_obj.int_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def int(self, w_obj):
+        assert w_obj is not None
         return w_obj.int(self)
 
+    @specialize.argtype(1)
     def uint_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.uint_w(self)
 
+    @specialize.argtype(1)
     def bigint_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return a rlib.rbigint object and call __long__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.bigint_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def float_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return an interp-level float and call __float__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.float_w(self, allow_conversion)
 
     def realtext_w(self, w_obj):
@@ -1733,7 +1747,9 @@
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
 
+    @specialize.argtype(1)
     def unicode_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.unicode_w(self)
 
     def unicode0_w(self, w_obj):
@@ -1758,7 +1774,9 @@
         # This is here mostly just for gateway.int_unwrapping_space_method().
         return bool(self.int_w(w_obj))
 
+    @specialize.argtype(1)
     def ord(self, w_obj):
+        assert w_obj is not None
         return w_obj.ord(self)
 
     # This is all interface for gateway.py.

From pypy.commits at gmail.com  Wed Sep  6 05:59:21 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 02:59:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: print an explanation why the function
 cannot be elidable/loopinvariant by
Message-ID: <59afc6f9.51bbdf0a.3b04e.a531@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92334:c2c1b6ca6794
Date: 2017-09-06 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/c2c1b6ca6794/

Log:	print an explanation why the function cannot be
	elidable/loopinvariant by showing the callstack leading to the
	problem

diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -185,6 +185,27 @@
                                          FUNC.RESULT, EffectInfo.MOST_GENERAL)
         return (fnaddr, calldescr)
 
+    def _raise_effect_error(self, op, extraeffect, functype, calling_graph):
+        explanation = []
+        if extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
+            explanation = self.randomeffects_analyzer.explain_analyze_slowly(op)
+        elif extraeffect == EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE:
+            explanation = self.virtualizable_analyzer.explain_analyze_slowly(op)
+        msg = []
+        if explanation:
+            msg = [
+                "_______ ERROR AT BOTTOM ______",
+                "RPython callstack leading to problem:",
+                ]
+            msg.extend(explanation)
+            msg.append("_______ ERROR: ______")
+        msg.append("operation %r" % op)
+        msg.append("in graph %s" % (calling_graph or "<unknown>"))
+        msg.append("this calls a %s function," % (functype, ))
+        msg.append(" but this contradicts other sources (e.g. it can have random"
+                   " effects): EF=%s" % (extraeffect, ))
+        raise Exception("\n".join(msg))
+
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
                      extraeffect=None, extradescr=None,
                      calling_graph=None):
@@ -278,18 +299,13 @@
         # check that the result is really as expected
         if loopinvariant:
             if extraeffect != EffectInfo.EF_LOOPINVARIANT:
-                raise Exception(
-                "operation %r in %s: this calls a _jit_loop_invariant_ function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, calling_graph, extraeffect))
+                self._raise_effect_error(op, extraeffect, "_jit_loop_invariant_", calling_graph)
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
                                    EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
-                raise Exception(
-                "operation %r in %s: this calls an elidable function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, calling_graph, extraeffect))
+
+                self._raise_effect_error(op, extraeffect, "elidable", calling_graph)
             elif RESULT is lltype.Void:
                 raise Exception(
                     "operation %r in %s: this calls an elidable function "
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -299,11 +299,23 @@
     def f4(n, m):
         return compute_hash(str(n) + str(m))
 
+    T = rffi.CArrayPtr(rffi.TIME_T)
+    external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True)
+
+    def effect():
+        return external(lltype.nullptr(T.TO))
+
+    @jit.elidable
+    def f5(n, m):
+        effect()
+        return 1
+
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
         d = f4(n, m)
+        f5(n, m)
         enable_siphash24()
         return a + len(b) + c + d
 
@@ -323,6 +335,14 @@
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.extraeffect == expected
 
+    call_op = f_graph.startblock.operations[4]
+    assert call_op.opname == 'direct_call'
+    excinfo = py.test.raises(Exception, cc.getcalldescr, call_op)
+    lines = excinfo.value.args[0].splitlines()
+    assert "f5" in lines[2]
+    assert "effect" in lines[3]
+    assert "random effects" in lines[-1]
+
 def test_raise_elidable_no_result():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
     l = []
diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py
--- a/rpython/translator/backendopt/graphanalyze.py
+++ b/rpython/translator/backendopt/graphanalyze.py
@@ -4,6 +4,7 @@
 
 class GraphAnalyzer(object):
     verbose = False
+    explanation = None
 
     def __init__(self, translator):
         self.translator = translator
@@ -73,6 +74,20 @@
     def compute_graph_info(self, graph):
         return None
 
+    def explain_analyze_slowly(self, op):
+        # this is a hack! usually done before a crash
+        self.__init__(self.translator)
+        self.explanation = explanation = []
+        oldverbose = self.verbose
+        self.verbose = True
+        try:
+            self.analyze(op)
+        finally:
+            del self.explanation
+            self.verbose = oldverbose
+        explanation.reverse()
+        return explanation
+
     def analyze(self, op, seen=None, graphinfo=None):
         if op.opname == "direct_call":
             try:
@@ -113,7 +128,11 @@
         return x
 
     def dump_info(self, info):
-        print '[%s] %s' % (self.__class__.__name__, info)
+        st = '[%s] %s' % (self.__class__.__name__, info)
+        if self.explanation is not None:
+            self.explanation.append(st)
+        else:
+            print st
 
     def analyze_direct_call(self, graph, seen=None):
         if seen is None:
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -531,3 +531,37 @@
         typed_effects = self._analyze_graph(t, wa, typed_write)
         typed_effects = self._filter_reads(typed_effects)
         assert typed_effects == direct_effects
+
+    def test_explanation(self):
+        class A(object):
+            def methodname(self):
+                self.x = 1
+                return 1
+            def m(self):
+                raise ValueError
+        class B(A):
+            def methodname(self):
+                return 2
+            def m(self):
+                return 3
+        def fancyname(a):
+            return a.methodname()
+        def m(a):
+            return a.m()
+        def h(flag):
+            if flag:
+                obj = A()
+            else:
+                obj = B()
+            fancyname(obj)
+            m(obj)
+
+        t, wa = self.translate(h, [int])
+        hgraph = graphof(t, h)
+        # fiiiish :-(
+        block = hgraph.startblock.exits[0].target.exits[0].target
+        op_call_fancyname = block.operations[0]
+
+        explanation = wa.explain_analyze_slowly(op_call_fancyname)
+        assert "fancyname" in explanation[0]
+        assert "methodname" in explanation[1]

From pypy.commits at gmail.com  Wed Sep  6 12:39:58 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 09:39:58 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: float coalescing support
Message-ID: <59b024de.8fb6df0a.eec0c.1686@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92338:f60a7734d1b4
Date: 2017-09-06 18:37 +0200
http://bitbucket.org/pypy/pypy/changeset/f60a7734d1b4/

Log:	float coalescing support

diff --git a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
@@ -77,17 +77,22 @@
     def fppii(x, y, i, j):
         return 19
 
+    def ff(x, y):
+        return x + y + 0.1
+
     F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed))
     F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed))
     F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed))
     FGCREFPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Signed))
     FPPIIPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF, lltype.Signed, lltype.Signed], lltype.Signed))
+    FFPTR = lltype.Ptr(lltype.FuncType([lltype.Float]*2, lltype.Float))
 
     f1ptr = llhelper(F1PTR, f1)
     f2ptr = llhelper(F2PTR, f2)
     f10ptr = llhelper(F10PTR, f10)
     fgcrefptr = llhelper(FGCREFPTR, fgcref)
     fppiiptr = llhelper(FPPIIPTR, fppii)
+    ffptr = llhelper(FFPTR, ff)
 
     f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT,
                                    EffectInfo.MOST_GENERAL)
@@ -99,6 +104,8 @@
                                        EffectInfo.MOST_GENERAL)
     fppii_calldescr = cpu.calldescrof(FPPIIPTR.TO, FPPIIPTR.TO.ARGS, FPPIIPTR.TO.RESULT,
                                       EffectInfo.MOST_GENERAL)
+    ff_calldescr = cpu.calldescrof(FFPTR.TO, FFPTR.TO.ARGS, FFPTR.TO.RESULT,
+                                   EffectInfo.MOST_GENERAL)
 
     namespace = locals().copy()
 
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -76,6 +76,8 @@
         else:
             self._consider_binop_symm(op, position)
 
+    consider_nursery_ptr_increment = consider_int_add
+
     def consider_int_sub(self, op, position):
         y = op.getarg(1)
         if isinstance(y, ConstInt) and rx86.fits_in_32bits(-y.value):
@@ -83,8 +85,17 @@
         else:
             self._consider_binop(op, position)
 
+    def _consider_float_op(self, op, position):
+        x = op.getarg(0)
+        if not isinstance(x, Const):
+            self.longevity.try_use_same_register(x, op)
 
-    consider_nursery_ptr_increment = consider_int_add
+    consider_float_add = _consider_float_op
+    consider_float_sub = _consider_float_op
+    consider_float_mul = _consider_float_op
+    consider_float_truediv = _consider_float_op
+    consider_float_neg = _consider_float_op
+    consider_float_abs = _consider_float_op
 
     def consider_int_lshift(self, op, position):
         x, y = op.getarg(0), op.getarg(1)
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -220,6 +220,19 @@
         # that would break coalescing between i7 and i9)
         assert op.args[1][0] is add1.args[-1]
 
+    def test_coalescing_float(self):
+        ops = '''
+        [f0, f1, f3]
+        f7 = float_add(f0, f1)
+        f8 = float_add(f7, f3)
+        f9 = call_f(ConstClass(ffptr), f8, 1.0, descr=ff_calldescr)
+        i10 = float_ne(f9, 0.0)
+        guard_true(i10) []
+        finish(f9)
+        '''
+        self.interpret(ops, [5.0, 6.0, 8.0])
+        assert len(self.filter_log_moves()) == 3
+
     def test_malloc(self, monkeypatch):
         ops = '''
         [i0]

From pypy.commits at gmail.com  Wed Sep  6 12:39:54 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 09:39:54 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: some cleanup in the call
 hinting implementation, a longer test with two calls
Message-ID: <59b024da.d05b1c0a.88fc1.bdd1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92336:364d9e356958
Date: 2017-09-06 18:08 +0200
http://bitbucket.org/pypy/pypy/changeset/364d9e356958/

Log:	some cleanup in the call hinting implementation, a longer test with
	two calls

diff --git a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc_integration.py
@@ -71,19 +71,34 @@
         assert len(args) == 10
         return sum(args)
 
+    def fgcref(x):
+        return 17
+
+    def fppii(x, y, i, j):
+        return 19
+
     F1PTR = lltype.Ptr(lltype.FuncType([lltype.Signed], lltype.Signed))
     F2PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*2, lltype.Signed))
     F10PTR = lltype.Ptr(lltype.FuncType([lltype.Signed]*10, lltype.Signed))
+    FGCREFPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF], lltype.Signed))
+    FPPIIPTR = lltype.Ptr(lltype.FuncType([llmemory.GCREF, llmemory.GCREF, lltype.Signed, lltype.Signed], lltype.Signed))
+
     f1ptr = llhelper(F1PTR, f1)
     f2ptr = llhelper(F2PTR, f2)
     f10ptr = llhelper(F10PTR, f10)
+    fgcrefptr = llhelper(FGCREFPTR, fgcref)
+    fppiiptr = llhelper(FPPIIPTR, fppii)
 
     f1_calldescr = cpu.calldescrof(F1PTR.TO, F1PTR.TO.ARGS, F1PTR.TO.RESULT,
                                    EffectInfo.MOST_GENERAL)
     f2_calldescr = cpu.calldescrof(F2PTR.TO, F2PTR.TO.ARGS, F2PTR.TO.RESULT,
                                    EffectInfo.MOST_GENERAL)
-    f10_calldescr= cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT,
-                                   EffectInfo.MOST_GENERAL)
+    f10_calldescr = cpu.calldescrof(F10PTR.TO, F10PTR.TO.ARGS, F10PTR.TO.RESULT,
+                                    EffectInfo.MOST_GENERAL)
+    fgcref_calldescr = cpu.calldescrof(FGCREFPTR.TO, FGCREFPTR.TO.ARGS, FGCREFPTR.TO.RESULT,
+                                       EffectInfo.MOST_GENERAL)
+    fppii_calldescr = cpu.calldescrof(FPPIIPTR.TO, FPPIIPTR.TO.ARGS, FPPIIPTR.TO.RESULT,
+                                      EffectInfo.MOST_GENERAL)
 
     namespace = locals().copy()
 
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -101,11 +101,6 @@
     consider_call_malloc_nursery_varsize = consider_call_malloc_nursery
     consider_call_malloc_nursery_varsize_frame = consider_call_malloc_nursery
 
-
-    def _call(self, op, position, args, save_all_regs=False):
-        # XXX fish for correct argtypes
-        CallHints64(self.longevity).hint(position, args, [], save_all_regs)
-
     def _consider_call(self, op, position, guard_not_forced=False, first_arg_index=1):
         calldescr = op.getdescr()
         assert isinstance(calldescr, CallDescr)
@@ -117,8 +112,9 @@
             gc_level = 1
         else:
             gc_level = 0
-        self._call(op, position, op.getarglist()[first_arg_index:],
-                   save_all_regs=gc_level)
+        args = op.getarglist()[first_arg_index:]
+        argtypes = calldescr.get_arg_types()
+        CallHints64(self.longevity).hint(position, args, argtypes, gc_level)
 
     def _consider_real_call(self, op, position):
         effectinfo = op.getdescr().get_extra_info()
@@ -148,10 +144,7 @@
 
     ARGUMENTS_GPR = [edi, esi, edx, ecx, r8, r9]
     ARGUMENTS_XMM = [xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7]
-    _ALL_CALLEE_SAVE_GPR = [ebx, r12, r13, r14, r15]
 
-    next_arg_gpr = 0
-    next_arg_xmm = 0
 
     def __init__(self, longevity):
         self.longevity = longevity
@@ -177,27 +170,26 @@
         hinted_xmm = []
         hinted_gpr = []
         hinted_args = []
+        next_arg_gpr = 0
+        next_arg_xmm = 0
         for i in range(len(args)):
             arg = args[i]
-            if arg.type == "f":
-                tgt = self._unused_xmm()
-                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
-                    self.longevity.fixed_register(position, tgt, arg)
-                    hinted_xmm.append(tgt)
-                    hinted_args.append(arg)
-            elif i < len(argtypes) and argtypes[i] == 'S':
-                # Singlefloat argument
-                tgt = self._unused_xmm()
-                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
-                    self.longevity.fixed_register(position, tgt, arg)
-                    hinted_xmm.append(tgt)
-                    hinted_args.append(arg)
+            if arg.type == "f" or (i < len(argtypes) and argtypes[i] == 'S'):
+                if next_arg_xmm < len(self.ARGUMENTS_XMM):
+                    tgt = self.ARGUMENTS_XMM[next_arg_xmm]
+                    if not arg.is_constant() and arg not in hinted_args:
+                        self.longevity.fixed_register(position, tgt, arg)
+                        hinted_xmm.append(tgt)
+                        hinted_args.append(arg)
+                    next_arg_xmm += 1
             else:
-                tgt = self._unused_gpr()
-                if tgt is not None and not arg.is_constant() and arg not in hinted_args:
-                    self.longevity.fixed_register(position, tgt, arg)
-                    hinted_gpr.append(tgt)
-                    hinted_args.append(arg)
+                if next_arg_gpr < len(self.ARGUMENTS_GPR):
+                    tgt = self.ARGUMENTS_GPR[next_arg_gpr]
+                    if not arg.is_constant() and arg not in hinted_args:
+                        self.longevity.fixed_register(position, tgt, arg)
+                        hinted_gpr.append(tgt)
+                        hinted_args.append(arg)
+                    next_arg_gpr += 1
         # block all remaining registers that are not caller save
         # XXX the case save_all_regs == 1 (save callee-save regs + gc ptrs) is
         # no expressible atm
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -49,7 +49,7 @@
                 arglocs, resloc, frame_depth)
 
     def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap):
-        self._log("malloc_cond", size) # always uses edx and ecx
+        self._log("malloc_cond", size, "ecx") # always uses edx and ecx
 
 
 class TestCheckRegistersExplicitly(test_regalloc_integration.BaseTestRegalloc):
@@ -259,3 +259,51 @@
         self.interpret(ops, [0], run=False)
         # 4 moves, three for args, one for result
         assert len(self.filter_log_moves()) == 4
+
+    def test_dict_lookup(self, monkeypatch):
+        monkeypatch.setattr(self.cpu.gc_ll_descr, "get_nursery_top_addr", lambda: 61)
+        monkeypatch.setattr(self.cpu.gc_ll_descr, "get_nursery_free_addr", lambda: 68)
+        # real trace for a dict lookup
+        ops = """
+        [i172, i182, i201, p209, p0, p219]
+        i184 = int_lt(i172, 0)
+        guard_false(i184) []
+        i185 = int_ge(i172, i182)
+        guard_false(i185) []
+        i187 = int_add(i172, 1)
+        i202 = uint_ge(i172, i201)
+        guard_false(i202) [i172]
+        i221 = int_xor(i172, 3430008)
+        i223 = int_mul(i221, 1000003)
+        i230 = call_i(ConstClass(fgcrefptr), p209, descr=fgcref_calldescr)
+        guard_no_exception() [p209, i230, i172]
+        i232 = int_eq(i230, -1)
+        i233 = int_sub(i230, i232)
+        i234 = int_xor(i223, i233)
+        i236 = int_mul(i234, 1082525)
+        i238 = int_add(i236, 97531)
+        i240 = int_eq(i238, -1)
+        i241 = int_sub(i238, i240)
+        p242 = force_token()
+        p244 = call_malloc_nursery(40)
+        gc_store(p244, 0, 83568, 8)
+        p249 = nursery_ptr_increment(p244, 24)
+        gc_store(p249, 0, 4656, 8)
+        gc_store(p249, 8, i172, 8)
+        #cond_call_gc_wb(p0)
+        gc_store(p0, 8, p242, 8)
+        i263 = call_may_force_i(ConstClass(fppiiptr), p219, p244, i241, 0, descr=fppii_calldescr)
+        guard_not_forced() [p0, p249, p244, i263, p219]
+        guard_no_exception() [p0, p249, p244, i263, p219]
+        i265 = int_lt(i263, 0)
+        guard_true(i265) [p0, p249, p244, i263, p219]
+        finish(i263)
+        """
+        self.interpret(ops, [0], run=False)
+        # the moves are:
+        # 5 arguments
+        # 1 result
+        # 1 because lifetime of i172 does not end at the int_xor
+        # 1 ptr to save before call
+        # 3 for argument shuffling
+        assert len(self.filter_log_moves()) == 11

From pypy.commits at gmail.com  Wed Sep  6 12:39:52 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 09:39:52 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: typo
Message-ID: <59b024d8.e2a9df0a.63bc4.2863@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92335:1af751546a21
Date: 2017-09-06 18:07 +0200
http://bitbucket.org/pypy/pypy/changeset/1af751546a21/

Log:	typo

diff --git a/rpython/jit/backend/llsupport/rewrite.py b/rpython/jit/backend/llsupport/rewrite.py
--- a/rpython/jit/backend/llsupport/rewrite.py
+++ b/rpython/jit/backend/llsupport/rewrite.py
@@ -354,7 +354,7 @@
             else:
                 # this is dead code, but in case we have a gc that does
                 # not have a write barrier and does not zero memory, we would
-                # need to clal it
+                # need to call it
                 if op.getopnum() == rop.SETFIELD_GC:
                     self.consider_setfield_gc(op)
                 elif op.getopnum() == rop.SETARRAYITEM_GC:

From pypy.commits at gmail.com  Wed Sep  6 14:27:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 06 Sep 2017 11:27:29 -0700 (PDT)
Subject: [pypy-commit] pypy pycheck-macros: add CPython-compatible
 Py*_Check{, Exact} macros to *.h
Message-ID: <59b03e11.05e11c0a.496a8.e05f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: pycheck-macros
Changeset: r92339:58c6a84c10a6
Date: 2017-09-03 00:54 +0300
http://bitbucket.org/pypy/pypy/changeset/58c6a84c10a6/

Log:	add CPython-compatible Py*_Check{,Exact} macros to *.h

diff --git a/pypy/module/cpyext/include/dictobject.h b/pypy/module/cpyext/include/dictobject.h
--- a/pypy/module/cpyext/include/dictobject.h
+++ b/pypy/module/cpyext/include/dictobject.h
@@ -12,6 +12,10 @@
     PyObject *_tmpkeys; /* a private place to put keys during PyDict_Next */
 } PyDictObject;
 
+#define PyDict_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_DICT_SUBCLASS)
+#define PyDict_CheckExact(op) ((op)->ob_type == &PyDict_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/intobject.h b/pypy/module/cpyext/include/intobject.h
--- a/pypy/module/cpyext/include/intobject.h
+++ b/pypy/module/cpyext/include/intobject.h
@@ -12,6 +12,10 @@
     long ob_ival;
 } PyIntObject;
 
+#define PyInt_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_INT_SUBCLASS)
+#define PyInt_CheckExact(op) ((op)->ob_type == &PyInt_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,4 @@
 /* empty */
+#define PyList_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS)
+#define PyList_CheckExact(op) ((op)->ob_type == &PyList_Type)
diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h
--- a/pypy/module/cpyext/include/longobject.h
+++ b/pypy/module/cpyext/include/longobject.h
@@ -14,6 +14,9 @@
 
 #define PyOS_strtoul strtoul
 #define PyOS_strtol strtoul
+#define PyLong_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS)
+#define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -236,6 +236,11 @@
 #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
 
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
+#define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
+
+#define PyType_Check(op) \
+    PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
+#define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type)
 
 /* objimpl.h ----------------------------------------------*/
 #define PyObject_New(type, typeobj) \
diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h
--- a/pypy/module/cpyext/include/pyerrors.h
+++ b/pypy/module/cpyext/include/pyerrors.h
@@ -9,7 +9,7 @@
 
 #define PyExceptionClass_Check(x)                                       \
     (PyClass_Check((x)) || (PyType_Check((x)) &&                        \
-      PyObject_IsSubclass((x), PyExc_BaseException)))
+      PyType_FastSubclass((PyTypeObject*)(x), Py_TPFLAGS_BASE_EXC_SUBCLASS)))
 
 PyAPI_FUNC(PyObject *) PyErr_NewException(const char *name, PyObject *base, PyObject *dict);
 PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict);
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -61,6 +61,10 @@
 PyAPI_FUNC(PyObject *) PyString_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyString_FromFormat(const char *format, ...);
 
+#define PyString_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_STRING_SUBCLASS)
+#define PyString_CheckExact(op) ((op)->ob_type == &PyString_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -26,6 +26,9 @@
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+#define PyTuple_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_TUPLE_SUBCLASS)
+#define PyTuple_CheckExact(op) ((op)->ob_type == &PyTuple_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,10 @@
 
 #include "cpyext_unicodeobject.h"
 
+#define PyUnicode_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS)
+#define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type)
+
 #ifdef __cplusplus
 }
 #endif

From pypy.commits at gmail.com  Wed Sep  6 14:27:33 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 06 Sep 2017 11:27:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix numpy test suite fail, test needed
Message-ID: <59b03e15.2eb0df0a.ced24.483f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92341:aa8394da03c4
Date: 2017-09-06 21:24 +0300
http://bitbucket.org/pypy/pypy/changeset/aa8394da03c4/

Log:	fix numpy test suite fail, test needed

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
@@ -940,6 +941,11 @@
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
                                                    space.newtext(message)))
+            except rstackovf.StackOverflow as e:
+                rstackovf.check_stack_overflow()
+                failed = True
+                state.set_exception(OperationError(space.w_RuntimeError,
+                         space.newtext("maximum recursion depth exceeded")))
             else:
                 failed = False
 

From pypy.commits at gmail.com  Wed Sep  6 14:27:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 06 Sep 2017 11:27:35 -0700 (PDT)
Subject: [pypy-commit] pypy pycheck-macros: merge default into branch
Message-ID: <59b03e17.5d87df0a.b73ef.4207@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: pycheck-macros
Changeset: r92342:568bae2b4fa4
Date: 2017-09-06 21:26 +0300
http://bitbucket.org/pypy/pypy/changeset/568bae2b4fa4/

Log:	merge default into branch

diff too long, truncating to 2000 out of 2013 lines

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,8 +119,15 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian and Ubuntu, this is the command to install all build-time
-dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -535,6 +535,11 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
+* In PyPy, module and class dictionaries are optimized under the assumption
+  that deleting attributes from them are rare. Because of this, e.g.
+  ``del foo.bar`` where ``foo`` is a module (or class) that contains the
+  function ``bar``, is significantly slower than CPython.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -73,3 +73,7 @@
 
 Add support for leakfinder in cpyext tests (disabled for now, due to too many
 failures).
+
+.. branch: pypy_swappedbytes
+
+Added ``_swappedbytes_`` support for ``ctypes.Structure``
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1629,12 +1629,15 @@
         # return text_w(w_obj) or None
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    @specialize.argtype(1)
     def bytes_w(self, w_obj):
         """ Takes an application level :py:class:`bytes`
             (on PyPy2 this equals `str`) and returns a rpython byte string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
+    @specialize.argtype(1)
     def text_w(self, w_obj):
         """ PyPy2 takes either a :py:class:`str` and returns a
             rpython byte string, or it takes an :py:class:`unicode`
@@ -1644,6 +1647,7 @@
             On PyPy3 it takes a :py:class:`str` and it will return
             an utf-8 encoded rpython string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1692,6 +1696,7 @@
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
         return chr(value)
 
+    @specialize.argtype(1)
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1704,26 +1709,35 @@
         If allow_conversion=False, w_obj needs to be an app-level int or a
         subclass.
         """
+        assert w_obj is not None
         return w_obj.int_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def int(self, w_obj):
+        assert w_obj is not None
         return w_obj.int(self)
 
+    @specialize.argtype(1)
     def uint_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.uint_w(self)
 
+    @specialize.argtype(1)
     def bigint_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return a rlib.rbigint object and call __long__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.bigint_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def float_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return an interp-level float and call __float__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.float_w(self, allow_conversion)
 
     def realtext_w(self, w_obj):
@@ -1733,7 +1747,9 @@
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
 
+    @specialize.argtype(1)
     def unicode_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.unicode_w(self)
 
     def unicode0_w(self, w_obj):
@@ -1758,7 +1774,9 @@
         # This is here mostly just for gateway.int_unwrapping_space_method().
         return bool(self.int_w(w_obj))
 
+    @specialize.argtype(1)
     def ord(self, w_obj):
+        assert w_obj is not None
         return w_obj.ord(self)
 
     # This is all interface for gateway.py.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
@@ -984,6 +985,11 @@
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
                                                    space.newtext(message)))
+            except rstackovf.StackOverflow as e:
+                rstackovf.check_stack_overflow()
+                failed = True
+                state.set_exception(OperationError(space.w_RuntimeError,
+                         space.newtext("maximum recursion depth exceeded")))
             else:
                 failed = False
 
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -136,7 +136,7 @@
     """Base class for all cpyext tests."""
     spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
                                    'itertools', 'time', 'binascii',
-                                   'micronumpy', 'mmap'
+                                   'mmap'
                                    ])
 
     @classmethod
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -125,52 +125,6 @@
         ten = foo.test_buffer(arr)
         assert ten == 10
 
-    @pytest.mark.skipif(only_pypy, reason='pypy only test')
-    def test_buffer_info(self):
-        try:
-            from _numpypy import multiarray as np
-        except ImportError:
-            skip('pypy built without _numpypy')
-        module = self.import_module(name='buffer_test')
-        get_buffer_info = module.get_buffer_info
-        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        arr = np.zeros((1, 10), order='F')
-        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-        assert strides[0] == 8
-        arr = np.zeros((10, 1), order='C')
-        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-        assert strides[-1] == 8
-        dt1 = np.dtype(
-             [('a', 'b'), ('b', 'i'),
-              ('sub0', np.dtype('b,i')),
-              ('sub1', np.dtype('b,i')),
-              ('sub2', np.dtype('b,i')),
-              ('sub3', np.dtype('b,i')),
-              ('sub4', np.dtype('b,i')),
-              ('sub5', np.dtype('b,i')),
-              ('sub6', np.dtype('b,i')),
-              ('sub7', np.dtype('b,i')),
-              ('c', 'i')],
-             )
-        x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
-        # pytest can catch warnings from v2.8 and up, we ship 2.5
-        import warnings
-        warnings.filterwarnings("error")
-        try:
-            try:
-                y = get_buffer_info(x, ['SIMPLE'])
-            except UserWarning as e:
-                pass
-            else:
-                assert False ,"PyPy-specific UserWarning not raised" \
-                          " on too long format string"
-        finally:
-            warnings.resetwarnings()
-        # calling get_buffer_info on x creates a memory leak,
-        # which is detected as an error at test teardown:
-        # Exception TypeError: "'NoneType' object is not callable"
-        #         in <bound method ConcreteArray.__del__ ...> ignored
-
     def test_releasebuffer(self):
         module = self.import_extension('foo', [
             ("create_test", "METH_NOARGS",
@@ -240,3 +194,55 @@
         self.debug_collect()
         assert module.get_cnt() == 0
         assert module.get_dealloc_cnt() == 1
+
+class AppTestBufferInfo(AppTestCpythonExtensionBase):
+    spaceconfig = AppTestCpythonExtensionBase.spaceconfig.copy()
+    spaceconfig['usemodules'].append('micronumpy')
+
+    @pytest.mark.skipif(only_pypy, reason='pypy only test')
+    def test_buffer_info(self):
+        try:
+            from _numpypy import multiarray as np
+        except ImportError:
+            skip('pypy built without _numpypy')
+        module = self.import_module(name='buffer_test')
+        get_buffer_info = module.get_buffer_info
+        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
+        arr = np.zeros((1, 10), order='F')
+        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
+        assert strides[0] == 8
+        arr = np.zeros((10, 1), order='C')
+        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
+        assert strides[-1] == 8
+        dt1 = np.dtype(
+             [('a', 'b'), ('b', 'i'),
+              ('sub0', np.dtype('b,i')),
+              ('sub1', np.dtype('b,i')),
+              ('sub2', np.dtype('b,i')),
+              ('sub3', np.dtype('b,i')),
+              ('sub4', np.dtype('b,i')),
+              ('sub5', np.dtype('b,i')),
+              ('sub6', np.dtype('b,i')),
+              ('sub7', np.dtype('b,i')),
+              ('c', 'i')],
+             )
+        x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
+        # pytest can catch warnings from v2.8 and up, we ship 2.5
+        import warnings
+        warnings.filterwarnings("error")
+        try:
+            try:
+                y = get_buffer_info(x, ['SIMPLE'])
+            except UserWarning as e:
+                pass
+            else:
+                assert False ,"PyPy-specific UserWarning not raised" \
+                          " on too long format string"
+        finally:
+            warnings.resetwarnings()
+        # calling get_buffer_info on x creates a memory leak,
+        # which is detected as an error at test teardown:
+        # Exception TypeError: "'NoneType' object is not callable"
+        #         in <bound method ConcreteArray.__del__ ...> ignored
+
+
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -26,6 +26,8 @@
 NULL = lltype.nullptr(rffi.VOIDP.TO)
 
 class TestNDArrayObject(BaseApiTest):
+    spaceconfig = AppTestCpythonExtensionBase.spaceconfig.copy()
+    spaceconfig['usemodules'].append('micronumpy')
 
     def test_Check(self, space, api):
         a = array(space, [10, 5, 3])
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -1,7 +1,7 @@
 import py
 
 
-class AppTestItertools: 
+class AppTestItertools(object):
     spaceconfig = dict(usemodules=['itertools'])
 
     def test_count(self):
@@ -330,11 +330,11 @@
 
     def test_chain(self):
         import itertools
-        
+
         it = itertools.chain()
         raises(StopIteration, it.next)
         raises(StopIteration, it.next)
-        
+
         it = itertools.chain([1, 2, 3])
         for x in [1, 2, 3]:
             assert it.next() == x
@@ -378,7 +378,7 @@
 
     def test_imap_wrongargs(self):
         import itertools
-        
+
         # Duplicate python 2.4 behaviour for invalid arguments
         it = itertools.imap(0, [])
         raises(StopIteration, it.next)
@@ -401,11 +401,11 @@
         for x in obj_list:
             assert it.next() == (x, )
         raises(StopIteration, it.next)
-        
+
         it = itertools.izip([1, 2, 3], [4], [5, 6])
         assert it.next() == (1, 4, 5)
         raises(StopIteration, it.next)
-        
+
         it = itertools.izip([], [], [1], [])
         raises(StopIteration, it.next)
 
@@ -423,7 +423,7 @@
 
     def test_izip_wrongargs(self):
         import itertools, re
-        
+
         # Duplicate python 2.4 behaviour for invalid arguments
         raises(TypeError, itertools.izip, None, 0)
 
@@ -442,7 +442,7 @@
 
         it = itertools.cycle([])
         raises(StopIteration, it.next)
-        
+
         it = itertools.cycle([1, 2, 3])
         for x in [1, 2, 3, 1, 2, 3, 1, 2, 3]:
             assert it.next() == x
@@ -498,7 +498,7 @@
 
     def test_tee_wrongargs(self):
         import itertools
-        
+
         raises(TypeError, itertools.tee, 0)
         raises(ValueError, itertools.tee, [], -1)
         raises(TypeError, itertools.tee, [], None)
@@ -536,7 +536,7 @@
 
     def test_groupby(self):
         import itertools
-        
+
         it = itertools.groupby([])
         raises(StopIteration, it.next)
 
@@ -613,7 +613,7 @@
             assert g.next() is x
             raises(StopIteration, g.next)
         raises(StopIteration, it.next)
-        
+
         # Grouping is based on key equality
         class AlwaysEqual(object):
             def __eq__(self, other):
@@ -647,7 +647,7 @@
 
     def test_iterables(self):
         import itertools
-    
+
         iterables = [
             itertools.chain(),
             itertools.count(),
@@ -665,7 +665,7 @@
             itertools.tee([])[0],
             itertools.tee([])[1],
             ]
-    
+
         for it in iterables:
             assert hasattr(it, '__iter__')
             assert iter(it) is it
@@ -674,7 +674,7 @@
 
     def test_docstrings(self):
         import itertools
-        
+
         assert itertools.__doc__
         methods = [
             itertools.chain,
@@ -756,15 +756,9 @@
         assert itertools.tee(a, 0) == ()
 
 
-class AppTestItertools26:
+class AppTestItertools26(object):
     spaceconfig = dict(usemodules=['itertools'])
 
-    def setup_class(cls):
-        if cls.space.is_true(cls.space.appexec([], """():
-            import sys; return sys.version_info < (2, 6)
-            """)):
-            py.test.skip("Requires Python 2.6")
-
     def test_count_overflow(self):
         import itertools, sys
         it = itertools.count(sys.maxint - 1)
@@ -1010,16 +1004,8 @@
         raises(ValueError, permutations, [1, 2], -1)
 
 
-class AppTestItertools27:
-    spaceconfig = {
-        "usemodules": ['itertools', 'struct', 'binascii'],
-    }
-
-    def setup_class(cls):
-        if cls.space.is_true(cls.space.appexec([], """():
-            import sys; return sys.version_info < (2, 7)
-            """)):
-            py.test.skip("Requires Python 2.7")
+class AppTestItertools27(object):
+    spaceconfig = {"usemodules": ['itertools', 'struct', 'binascii']}
 
     def test_compress(self):
         import itertools
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -116,10 +116,10 @@
     if check_int_or_float:
         for w_obj in list_w:
             if type(w_obj) is W_IntObject:
-                if longlong2float.can_encode_int32(space.int_w(w_obj)):
+                if longlong2float.can_encode_int32(w_obj.int_w(space)):
                     continue    # ok
             elif type(w_obj) is W_FloatObject:
-                if longlong2float.can_encode_float(space.float_w(w_obj)):
+                if longlong2float.can_encode_float(w_obj.float_w(space)):
                     continue    # ok
             break
         else:
diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -31,23 +31,23 @@
     class cls(W_AbstractTupleObject):
         _immutable_fields_ = ['value%s' % i for i in iter_n]
 
-        def __init__(self, space, *values_w):
+        def __init__(self, space, *values):
             self.space = space
-            assert len(values_w) == typelen
+            assert len(values) == typelen
             for i in iter_n:
-                w_obj = values_w[i]
+                obj = values[i]
                 val_type = typetuple[i]
                 if val_type == int:
-                    unwrapped = w_obj.int_w(space)
+                    assert isinstance(obj, int)
                 elif val_type == float:
-                    unwrapped = w_obj.float_w(space)
+                    assert isinstance(obj, float)
                 elif val_type == str:
-                    unwrapped = w_obj.str_w(space)
+                    assert isinstance(obj, str)
                 elif val_type == object:
-                    unwrapped = w_obj
+                    pass
                 else:
                     raise AssertionError
-                setattr(self, 'value%s' % i, unwrapped)
+                setattr(self, 'value%s' % i, obj)
 
         def length(self):
             return typelen
@@ -150,10 +150,10 @@
         w_arg1, w_arg2 = list_w
         if type(w_arg1) is W_IntObject:
             if type(w_arg2) is W_IntObject:
-                return Cls_ii(space, w_arg1, w_arg2)
+                return Cls_ii(space, space.int_w(w_arg1), space.int_w(w_arg2))
         elif type(w_arg1) is W_FloatObject:
             if type(w_arg2) is W_FloatObject:
-                return Cls_ff(space, w_arg1, w_arg2)
+                return Cls_ff(space, space.float_w(w_arg1), space.float_w(w_arg2))
         return Cls_oo(space, w_arg1, w_arg2)
     else:
         raise NotSpecialised
@@ -170,10 +170,9 @@
 # faster to move the decision out of the loop.
 
 @specialize.arg(1)
-def _build_zipped_spec(space, Cls, lst1, lst2, wrap1, wrap2):
+def _build_zipped_spec(space, Cls, lst1, lst2):
     length = min(len(lst1), len(lst2))
-    return [Cls(space, wrap1(lst1[i]),
-                       wrap2(lst2[i])) for i in range(length)]
+    return [Cls(space, lst1[i], lst2[i]) for i in range(length)]
 
 def _build_zipped_spec_oo(space, w_list1, w_list2):
     strat1 = w_list1.strategy
@@ -200,8 +199,7 @@
             intlist2 = w_list2.getitems_int()
             if intlist2 is not None:
                 lst_w = _build_zipped_spec(
-                        space, Cls_ii, intlist1, intlist2,
-                        space.newint, space.newint)
+                        space, Cls_ii, intlist1, intlist2)
                 return space.newlist(lst_w)
         else:
             floatlist1 = w_list1.getitems_float()
@@ -209,8 +207,7 @@
                 floatlist2 = w_list2.getitems_float()
                 if floatlist2 is not None:
                     lst_w = _build_zipped_spec(
-                        space, Cls_ff, floatlist1, floatlist2, space.newfloat,
-                        space.newfloat)
+                        space, Cls_ff, floatlist1, floatlist2)
                     return space.newlist(lst_w)
 
         lst_w = _build_zipped_spec_oo(space, w_list1, w_list2)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4644,6 +4644,25 @@
         s_exc = a.binding(graphof(a, f).exceptblock.inputargs[1])
         assert not s_exc.can_be_none()
 
+    def test_specialize_argtype_with_subclasses(self):
+        # checks that specialize:argtype() makes two copies of a
+        # function f(), one for the base class and one for the subclass
+        class A:
+            def foo(self):
+                return 123
+        class B(A):
+            def foo(self):
+                return 456
+        def f(x):
+            return x.foo()
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            if y > 5:
+                f(A())
+            return f(B())
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]).const == 456
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -185,8 +185,30 @@
                                          FUNC.RESULT, EffectInfo.MOST_GENERAL)
         return (fnaddr, calldescr)
 
+    def _raise_effect_error(self, op, extraeffect, functype, calling_graph):
+        explanation = []
+        if extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
+            explanation = self.randomeffects_analyzer.explain_analyze_slowly(op)
+        elif extraeffect == EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE:
+            explanation = self.virtualizable_analyzer.explain_analyze_slowly(op)
+        msg = []
+        if explanation:
+            msg = [
+                "_______ ERROR AT BOTTOM ______",
+                "RPython callstack leading to problem:",
+                ]
+            msg.extend(explanation)
+            msg.append("_______ ERROR: ______")
+        msg.append("operation %r" % op)
+        msg.append("in graph %s" % (calling_graph or "<unknown>"))
+        msg.append("this calls a %s function," % (functype, ))
+        msg.append(" but this contradicts other sources (e.g. it can have random"
+                   " effects): EF=%s" % (extraeffect, ))
+        raise Exception("\n".join(msg))
+
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None,
+                     calling_graph=None):
         """Return the calldescr that describes all calls done by 'op'.
         This returns a calldescr that we can put in the corresponding
         call operation in the calling jitcode.  It gets an effectinfo
@@ -202,13 +224,13 @@
         ARGS = FUNC.ARGS
         if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
+                "operation %r in %s: calling a function with signature %r, "
                 "but passing actual arguments (ignoring voids) of types %r"
-                % (op, FUNC, NON_VOID_ARGS))
+                % (op, calling_graph, FUNC, NON_VOID_ARGS))
         if RESULT != FUNC.RESULT:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
-                "but the actual return type is %r" % (op, FUNC, RESULT))
+                "%r in %s: calling a function with signature %r, "
+                "but the actual return type is %r" % (op, calling_graph, FUNC, RESULT))
         # ok
         # get the 'elidable' and 'loopinvariant' flags from the function object
         elidable = False
@@ -217,7 +239,7 @@
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
-                "%r: getcalldescr() with a non-default call ABI" % (op,))
+                "%r in %s: getcalldescr() with a non-default call ABI" % (op, calling_graph))
             func = getattr(funcobj, '_callable', None)
             elidable = getattr(func, "_elidable_function_", False)
             loopinvariant = getattr(func, "_jit_loop_invariant_", False)
@@ -245,11 +267,11 @@
                 if not error:
                     continue
                 raise Exception(
-                    "%r is an indirect call to a family of functions "
+                    "%r in %s is an indirect call to a family of functions "
                     "(or methods) that includes %r. However, the latter "
                     "is marked %r. You need to use an indirection: replace "
                     "it with a non-marked function/method which calls the "
-                    "marked function." % (op, graph, error))
+                    "marked function." % (op, calling_graph, graph, error))
         # build the extraeffect
         random_effects = self.randomeffects_analyzer.analyze(op)
         if random_effects:
@@ -277,22 +299,17 @@
         # check that the result is really as expected
         if loopinvariant:
             if extraeffect != EffectInfo.EF_LOOPINVARIANT:
-                raise Exception(
-                "in operation %r: this calls a _jit_loop_invariant_ function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+                self._raise_effect_error(op, extraeffect, "_jit_loop_invariant_", calling_graph)
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
                                    EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
-                raise Exception(
-                "in operation %r: this calls an elidable function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+
+                self._raise_effect_error(op, extraeffect, "elidable", calling_graph)
             elif RESULT is lltype.Void:
                 raise Exception(
-                    "in operation %r: this calls an elidable function "
-                    "but the function has no result" % (op, ))
+                    "operation %r in %s: this calls an elidable function "
+                    "but the function has no result" % (op, calling_graph))
         #
         effectinfo = effectinfo_from_writeanalyze(
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -64,6 +64,7 @@
         self.cpu = cpu
         self.callcontrol = callcontrol
         self.portal_jd = portal_jd   # non-None only for the portal graph(s)
+        self.graph = None
 
     def transform(self, graph):
         self.graph = graph
@@ -424,7 +425,8 @@
         of 'residual_call_xxx' are the function to call, and its calldescr."""
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex=oopspecindex,
                                                   extraeffect=extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         op1 = self.rewrite_call(op, 'residual_call',
                                 [op.args[0]] + extraargs, calldescr=calldescr)
         if may_call_jitcodes or self.callcontrol.calldescr_canraise(calldescr):
@@ -1613,7 +1615,9 @@
         if len(op.args) > 4 + 2 or have_floats:
             raise Exception("Conditional call does not support floats or more than 4 arguments")
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
-        calldescr = self.callcontrol.getcalldescr(callop)
+        calldescr = self.callcontrol.getcalldescr(
+                callop,
+                calling_graph=self.graph)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
@@ -1924,7 +1928,8 @@
                              extradescr=None):
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
                                                   extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         if extraeffect is not None:
             assert (is_test_calldescr(calldescr)      # for tests
                     or calldescr.get_extra_info().extraeffect == extraeffect)
@@ -1954,7 +1959,8 @@
                             [c_func] + [varoftype(T) for T in argtypes],
                             varoftype(resulttype))
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
-                                                  effectinfo)
+                                                  effectinfo,
+                                                  calling_graph=self.graph)
         if isinstance(c_func.value, str):    # in tests only
             func = c_func.value
         else:
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -299,11 +299,23 @@
     def f4(n, m):
         return compute_hash(str(n) + str(m))
 
+    T = rffi.CArrayPtr(rffi.TIME_T)
+    external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True)
+
+    def effect():
+        return external(lltype.nullptr(T.TO))
+
+    @jit.elidable
+    def f5(n, m):
+        effect()
+        return 1
+
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
         d = f4(n, m)
+        f5(n, m)
         enable_siphash24()
         return a + len(b) + c + d
 
@@ -323,25 +335,33 @@
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.extraeffect == expected
 
+    call_op = f_graph.startblock.operations[4]
+    assert call_op.opname == 'direct_call'
+    excinfo = py.test.raises(Exception, cc.getcalldescr, call_op)
+    lines = excinfo.value.args[0].splitlines()
+    assert "f5" in lines[2]
+    assert "effect" in lines[3]
+    assert "random effects" in lines[-1]
+
 def test_raise_elidable_no_result():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
     l = []
     @jit.elidable
     def f1(n, m):
         l.append(n)
-    def f(n, m):
+    def fancy_graph_name(n, m):
         f1(n, m)
         return n + m
 
-    rtyper = support.annotate(f, [7, 9])
+    rtyper = support.annotate(fancy_graph_name, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
     cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
     res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
+    [f_graph] = [x for x in res if x.func is fancy_graph_name]
     call_op = f_graph.startblock.operations[0]
     assert call_op.opname == 'direct_call'
-    with py.test.raises(Exception):
-        call_descr = cc.getcalldescr(call_op)
+    x = py.test.raises(Exception, cc.getcalldescr, call_op, calling_graph=f_graph)
+    assert "fancy_graph_name" in str(x.value)
 
 def test_can_or_cannot_collect():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -73,7 +73,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None, calling_graph=None):
         try:
             name = op.args[0].value._obj._name
             if 'cannot_raise' in name or name.startswith('cast_'):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -48,7 +48,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         return 'calldescr'
     def calldescr_canraise(self, calldescr):
         return True
@@ -106,7 +106,7 @@
     def guess_call_kind(self, op):
         return 'builtin'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         assert oopspecindex is not None    # in this test
         EI = effectinfo.EffectInfo
         if oopspecindex != EI.OS_ARRAYCOPY:
diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py
--- a/rpython/jit/codewriter/test/test_list.py
+++ b/rpython/jit/codewriter/test/test_list.py
@@ -39,7 +39,7 @@
 class FakeCallControl:
     class getcalldescr(AbstractDescr):
         def __init__(self, op, oopspecindex=0, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
             self.op = op
             self.oopspecindex = oopspecindex
         def __repr__(self):
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -329,11 +329,14 @@
 
     def make_guards(self, op, short, optimizer):
         if self._known_class is not None:
-            short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
+                short.append(ResOperation(rop.GUARD_NONNULL, [op]))
                 short.append(ResOperation(rop.GUARD_IS_OBJECT, [op]))
-            short.append(ResOperation(rop.GUARD_CLASS,
-                                      [op, self._known_class]))
+                short.append(ResOperation(rop.GUARD_CLASS,
+                                          [op, self._known_class]))
+            else:
+                short.append(ResOperation(rop.GUARD_NONNULL_CLASS,
+                    [op, self._known_class]))
         elif self.descr is not None:
             short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -7541,6 +7541,33 @@
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
+    def test_guards_before_getfields_in_short_preamble_removetypeptr(self, monkeypatch):
+        monkeypatch.setattr(self.cpu, "remove_gctypeptr", True)
+        ops = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p0)
+        """
+        expected = """
+        [p0, p1]
+        jump(p0, p1)
+        """
+        short = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
+
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -2,6 +2,7 @@
 from rpython.rtyper.lltypesystem import lltype
 from rpython.rtyper.tool import rffi_platform as platform
 from rpython.rtyper.lltypesystem.rffi import CCHARP
+from rpython.rlib import jit
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
 from rpython.translator.platform import platform as target_platform
 
@@ -190,6 +191,8 @@
 
 IPX_TYPE
 
+SCM_RIGHTS
+
 POLLIN POLLPRI POLLOUT POLLERR POLLHUP POLLNVAL
 POLLRDNORM POLLRDBAND POLLWRNORM POLLWEBAND POLLMSG
 
@@ -260,6 +263,7 @@
 sockaddr_ptr = lltype.Ptr(lltype.ForwardReference())
 addrinfo_ptr = lltype.Ptr(lltype.ForwardReference())
 
+
 # struct types
 CConfig.sockaddr = platform.Struct('struct sockaddr',
                                              [('sa_family', rffi.INT),
@@ -343,6 +347,650 @@
                                 [('ifr_ifindex', rffi.INT),
                                  ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
+# insert handler for sendmsg / recvmsg here
+if _POSIX:
+    includes = ['stddef.h',
+                'sys/socket.h',
+                'unistd.h',
+                'string.h',
+                'stdlib.h',
+                'errno.h',
+                'limits.h',
+                'stdio.h',
+                'sys/types.h',
+                'netinet/in.h',
+                'arpa/inet.h']
+    separate_module_sources = ['''
+
+        // special defines for returning from recvmsg
+        #define BAD_MSG_SIZE_GIVEN -10000
+        #define BAD_ANC_SIZE_GIVEN -10001
+        #define MAL_ANC -10002
+
+        // special defines for returning from sendmsg
+        #define MUL_MSGS_NOT_SUP -1000
+        #define ANC_DATA_TOO_LARGE -1001
+        #define ANC_DATA_TOO_LARGEX -1002
+
+        /*
+            Even though you could, theoretically, receive more than one message, IF you set the socket option,
+            CPython has hardcoded the message number to 1, and implemented the option to receive more then 1 in a
+            different socket method: recvmsg_into
+        */
+        #define MSG_IOVLEN 1 // CPython has hardcoded this as well.
+        #if INT_MAX > 0x7fffffff
+            #define SOCKLEN_T_LIMIT 0x7fffffff
+        #else
+        #define SOCKLEN_T_LIMIT INT_MAX
+        #endif
+
+        // ################################################################################################
+        // Recvmsg implementation and associated functions
+
+        // Taken from CPython. Determines the minimum memory space required for the ancillary data.
+        #ifdef CMSG_SPACE
+        static int
+        cmsg_min_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t space)
+        {
+            size_t cmsg_offset;
+            static const size_t cmsg_len_end = (offsetof(struct cmsghdr, cmsg_len) +
+                                                sizeof(cmsgh->cmsg_len));
+
+            /* Note that POSIX allows msg_controllen to be of signed type. */
+            if (cmsgh == NULL || msg->msg_control == NULL)
+                return 0;
+            /* Note that POSIX allows msg_controllen to be of a signed type. This is
+               annoying under OS X as it's unsigned there and so it triggers a
+               tautological comparison warning under Clang when compared against 0.
+               Since the check is valid on other platforms, silence the warning under
+               Clang. */
+            #ifdef __clang__
+            #pragma clang diagnostic push
+            #pragma clang diagnostic ignored "-Wtautological-compare"
+            #endif
+            #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+            #pragma GCC diagnostic push
+            #pragma GCC diagnostic ignored "-Wtype-limits"
+            #endif
+            if (msg->msg_controllen < 0)
+                return 0;
+            #if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+            #pragma GCC diagnostic pop
+            #endif
+            #ifdef __clang__
+            #pragma clang diagnostic pop
+            #endif
+            if (space < cmsg_len_end)
+                space = cmsg_len_end;
+            cmsg_offset = (char *)cmsgh - (char *)msg->msg_control;
+            return (cmsg_offset <= (size_t)-1 - space &&
+                    cmsg_offset + space <= msg->msg_controllen);
+        }
+        #endif
+
+        // Taken from CPython.
+        #ifdef CMSG_LEN
+        /* If pointer CMSG_DATA(cmsgh) is in buffer msg->msg_control, set
+           *space to number of bytes following it in the buffer and return
+           true; otherwise, return false.  Assumes cmsgh, msg->msg_control and
+           msg->msg_controllen are valid. */
+        static int
+        get_cmsg_data_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *space)
+        {
+            size_t data_offset;
+            char *data_ptr;
+
+            if ((data_ptr = (char *)CMSG_DATA(cmsgh)) == NULL)
+                return 0;
+            data_offset = data_ptr - (char *)msg->msg_control;
+            if (data_offset > msg->msg_controllen)
+                return 0;
+            *space = msg->msg_controllen - data_offset;
+            return 1;
+        }
+
+        // Taken from CPython.
+        /* If cmsgh is invalid or not contained in the buffer pointed to by
+           msg->msg_control, return -1.  If cmsgh is valid and its associated
+           data is entirely contained in the buffer, set *data_len to the
+           length of the associated data and return 0.  If only part of the
+           associated data is contained in the buffer but cmsgh is otherwise
+           valid, set *data_len to the length contained in the buffer and
+           return 1. */
+        static int
+        get_cmsg_data_len(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *data_len)
+        {
+            size_t space, cmsg_data_len;
+
+            if (!cmsg_min_space(msg, cmsgh, CMSG_LEN(0)) ||
+                cmsgh->cmsg_len < CMSG_LEN(0))
+                return -1;
+            cmsg_data_len = cmsgh->cmsg_len - CMSG_LEN(0);
+            if (!get_cmsg_data_space(msg, cmsgh, &space))
+                return -1;
+            if (space >= cmsg_data_len) {
+                *data_len = cmsg_data_len;
+                return 0;
+            }
+            *data_len = space;
+            return 1;
+        }
+        #endif    /* CMSG_LEN */
+
+        /*
+            Structure meant to hold the information received after a recvmsg is performed.
+            Essentially it holds: the address, the message, the ancillary data and the return flags.
+            I use this structure for 2 main reasons:
+               - keep things ordered
+               - some of the ancillary parameters need to be int not long (rffi SignedP is actually long*),
+                 therefore I cannot use the parameters directly
+        */
+        struct recvmsg_info
+        {
+            struct sockaddr* address; // address fields
+            socklen_t addrlen;
+            int* length_of_messages; // message fields
+            char** messages;
+            int no_of_messages;
+            int size_of_ancillary; // ancillary fields
+            int* levels;
+            int* types;
+            char** file_descr;
+            int* descr_per_ancillary;
+            int retflag; // return flag field
+        };
+
+        /*
+            Wrapper function over recvmsg. Since it returns a lot of data,
+            in a structure that is hard to parse in rffi, it was implemented in C.
+            All the parameters, save the socket fd, message_size, ancillary_size
+            will be malloc'd and/or modified.
+        */
+        RPY_EXTERN
+        int recvmsg_implementation(
+                                  int socket_fd,
+                                  int message_size,
+                                  int ancillary_size,
+                                  int flags,
+                                  struct sockaddr* address,
+                                  socklen_t* addrlen,
+                                  long** length_of_messages,
+                                  char** messages,
+                                  long* no_of_messages,
+                                  long* size_of_ancillary,
+                                  long** levels,
+                                  long** types,
+                                  char** file_descr,
+                                  long** descr_per_ancillary,
+                                  long* retflag)
+
+        {
+
+            struct sockaddr* recvd_address;
+            socklen_t recvd_addrlen;
+            struct msghdr msg = {0};
+            void *controlbuf = NULL;
+            struct cmsghdr *cmsgh;
+            int cmsg_status;
+            struct iovec iov;
+            struct recvmsg_info* retinfo;
+            int error_flag;   // variable to be set in case of special errors.
+            int cmsgdatalen = 0;
+
+            // variables that are set to 1, if the message charp has been allocated
+            // and if the ancillary variables have been allocated. To be used in case of failure.
+            int iov_alloc = 0;
+            int anc_alloc = 0;
+
+            retinfo = (struct recvmsg_info*) malloc(sizeof(struct recvmsg_info));
+
+            if (ancillary_size > SOCKLEN_T_LIMIT){
+                error_flag = BAD_ANC_SIZE_GIVEN;
+                goto fail;
+            }
+
+            // Setup the messages iov struct memory
+            iov.iov_base = (char*) malloc(message_size);
+            memset(iov.iov_base, 0, message_size);
+            iov.iov_len = message_size;
+
+            // Setup the ancillary buffer memory
+            controlbuf = malloc(ancillary_size);
+
+            // Setup the recv address memory
+            recvd_addrlen = sizeof(struct sockaddr_storage);
+            recvd_address = (struct sockaddr*) malloc(recvd_addrlen);
+
+            memset(recvd_address, 0,recvd_addrlen);
+
+            // Setup the msghdr struct
+            msg.msg_name = recvd_address;
+            msg.msg_namelen = recvd_addrlen;
+            msg.msg_iov = &iov;
+            msg.msg_iovlen = MSG_IOVLEN;
+            msg.msg_control = controlbuf;
+            msg.msg_controllen = ancillary_size;
+
+            // Link my structure to the msghdr fields
+            retinfo->address = msg.msg_name;
+            retinfo->length_of_messages = (int*) malloc (MSG_IOVLEN * sizeof(int));
+            retinfo->no_of_messages = MSG_IOVLEN;
+            retinfo->messages = (char**) malloc (MSG_IOVLEN * sizeof(char*));
+            retinfo->messages[0] = msg.msg_iov->iov_base;
+
+            iov_alloc = 1;
+            ssize_t bytes_recvd = 0;
+
+            bytes_recvd = recvmsg(socket_fd, &msg, flags);
+
+            if (bytes_recvd < 0){
+                 goto fail;
+            }
+
+            retinfo->addrlen = (socklen_t) msg.msg_namelen;
+            retinfo->length_of_messages[0] = msg.msg_iov->iov_len;
+
+            // Count the ancillary items & allocate the memory
+            int anc_counter = 0;
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+
+                 anc_counter++;
+            }
+            retinfo->size_of_ancillary = anc_counter;
+            retinfo->file_descr = (char**) malloc (anc_counter * sizeof(char*));
+            retinfo->levels = (int*) malloc(anc_counter * sizeof(int));
+            retinfo->types = (int*) malloc(anc_counter * sizeof(int));
+            retinfo->descr_per_ancillary = (int*) malloc(anc_counter * sizeof(int));
+            anc_alloc = 1;
+
+            // Extract the ancillary items
+            int i=0;
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+                 size_t local_size = 0;
+                 cmsg_status = get_cmsg_data_len(&msg, cmsgh, &local_size);
+                 if (cmsg_status !=0 ){
+                    error_flag = MAL_ANC;
+                    goto err_closefds;
+                 }
+                 retinfo->file_descr[i] = (char*) malloc(local_size);
+                 memcpy(retinfo->file_descr[i], CMSG_DATA(cmsgh), local_size);
+                 retinfo->levels[i] = cmsgh->cmsg_level;
+                 retinfo->types[i] = cmsgh->cmsg_type;
+                 retinfo->descr_per_ancillary[i] =local_size;
+                 i++;
+
+            }
+            retinfo->retflag = msg.msg_flags;
+
+            // Set the parameters of address
+            memcpy(address,retinfo->address,retinfo->addrlen);
+            *addrlen = retinfo->addrlen;
+
+            // Set the parameters of message
+            no_of_messages[0] = retinfo->no_of_messages;
+            size_of_ancillary[0] = retinfo->size_of_ancillary;
+            *length_of_messages = (long*) malloc (sizeof(long) * retinfo->no_of_messages);
+            //memcpy(*length_of_messages, retinfo->length_of_messages, sizeof(int) * retinfo->no_of_messages);
+            int counter = 0;
+            for (i=0; i< retinfo->no_of_messages; i++){
+                counter += retinfo->length_of_messages[i];
+                length_of_messages[0][i] = retinfo->length_of_messages[i];
+            }
+            memset(*messages, 0, sizeof(char) * counter);
+            counter = 0;
+            for(i=0; i< retinfo->no_of_messages; i++){
+                memcpy(*messages+counter,retinfo->messages[i],retinfo->length_of_messages[i]);
+                counter += retinfo->length_of_messages[i];
+            }
+
+            // Set the parameters of ancillary
+            *levels = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            *types = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            *descr_per_ancillary = (long*) malloc (sizeof(long) * retinfo->size_of_ancillary);
+            counter = 0;
+            for (i=0; i < retinfo->size_of_ancillary; i++){
+                counter += retinfo->descr_per_ancillary[i];
+                // Convert the int* to long*
+                levels[0][i] = (long) retinfo->levels[i];
+                types[0][i] = (long) retinfo->types[i];
+                descr_per_ancillary[0][i] = (long) retinfo->descr_per_ancillary[i];
+            }
+            *file_descr = (char*) malloc (sizeof(char) * counter);
+            memset(*file_descr, 0, sizeof(char) * counter);
+            counter = 0;
+            for (i=0; i<retinfo->size_of_ancillary; i++){
+                memcpy(*file_descr+counter,retinfo->file_descr[i], retinfo->descr_per_ancillary[i]);
+                counter += retinfo->descr_per_ancillary[i];
+            }
+
+            // Set the retflag
+            retflag[0] = retinfo->retflag;
+
+            // Free the memory
+            free(retinfo->address);
+            free(retinfo->length_of_messages);
+            free(retinfo->levels);
+            free(retinfo->types);
+            free(retinfo->descr_per_ancillary);
+            for(i = 0; i<retinfo->no_of_messages; i++)
+                free(retinfo->messages[i]);
+            for (i = 0; i < retinfo->size_of_ancillary; i++)
+                free(retinfo->file_descr[i]);
+            free(retinfo->file_descr);
+            free(retinfo->messages);
+            free(retinfo);
+            free(controlbuf);
+
+            return bytes_recvd;
+
+        fail:
+            if (anc_alloc){
+                free(retinfo->file_descr);
+                free(retinfo->levels);
+                free(retinfo->types);
+                free(retinfo->descr_per_ancillary);
+                free(retinfo->length_of_messages);
+                free(retinfo->messages[0]);
+                free(retinfo->messages);
+                free(retinfo->address);
+                free(retinfo);
+                free(controlbuf);
+
+            }else{
+                if (iov_alloc){
+                    free(retinfo->length_of_messages);
+                    free(retinfo->messages[0]);
+                    free(retinfo->messages);
+                    free(retinfo->address);
+                    free(controlbuf);
+                    free(retinfo);
+                }
+            }
+            return error_flag;
+
+        err_closefds:
+        // Special case for UNIX sockets. In case file descriptors are received, they need to be closed.
+        // Taken from CPython
+        #ifdef SCM_RIGHTS
+            /* Close all descriptors coming from SCM_RIGHTS, so they don't leak. */
+            for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL);
+                 cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) {
+                size_t dataleng;
+                cmsg_status = get_cmsg_data_len(&msg, cmsgh, &dataleng);
+                cmsgdatalen = (int) dataleng;
+                if (cmsg_status < 0)
+                    break;
+                if (cmsgh->cmsg_level == SOL_SOCKET &&
+                    cmsgh->cmsg_type == SCM_RIGHTS) {
+                    size_t numfds;
+                    int *fdp;
+
+                    numfds = cmsgdatalen / sizeof(int);
+                    fdp = (int *)CMSG_DATA(cmsgh);
+                    while (numfds-- > 0)
+                        close(*fdp++);
+                }
+                if (cmsg_status != 0)
+                    break;
+            }
+        #endif /* SCM_RIGHTS */
+            goto fail;
+        }
+
+
+        // ################################################################################################
+        // Sendmsg implementation and associated functions
+
+        #ifdef CMSG_LEN
+        static int
+        get_CMSG_LEN(size_t length, size_t *result)
+        {
+            size_t tmp;
+
+            if (length > (SOCKLEN_T_LIMIT - CMSG_LEN(0)))
+                return 0;
+            tmp = CMSG_LEN(length);
+            if ((tmp > SOCKLEN_T_LIMIT) || (tmp < length))
+                return 0;
+            *result = tmp;
+            return 1;
+        }
+        #endif
+
+        #ifdef CMSG_SPACE
+        /* If length is in range, set *result to CMSG_SPACE(length) and return
+           true; otherwise, return false. */
+        static int
+        get_CMSG_SPACE(size_t length, size_t *result)
+        {
+            size_t tmp;
+
+            /* Use CMSG_SPACE(1) here in order to take account of the padding
+               necessary before *and* after the data. */
+            if (length > (SOCKLEN_T_LIMIT - CMSG_SPACE(1)))
+                return 0;
+            tmp = CMSG_SPACE(length);
+            if ((tmp > SOCKLEN_T_LIMIT) || (tmp < length))
+                return 0;
+            *result = tmp;
+            return 1;
+        }
+        #endif
+
+        /*
+            sendmsg_implementation is a wrapper over sendmsg of the API.
+            It was inspired from the way CPython did their implementation of this.
+            The main reason that it was written in C, is the struct msghdr,
+            which contains the ancillary data in a linked list of cmsghdr structures.
+            It was simpler to use it in C, and then push the simpler types of data via rffi.
+        */
+        RPY_EXTERN
+        int sendmsg_implementation
+                            (int socket,
+                            struct sockaddr* address,
+                            socklen_t addrlen,
+                            long* length_of_messages,
+                            char** messages,
+                            int no_of_messages,
+                            long* levels,
+                            long* types,
+                            char** file_descriptors,
+                            long* no_of_fds,
+                            int control_length,
+                            int flag
+                            )
+        {
+
+            struct msghdr        msg = {0};
+            struct cmsghdr       *cmsg;
+            void* controlbuf = NULL;
+            int retval;
+            size_t i;
+
+            // Prepare the msghdr structure for the send:
+
+            // Add the address
+            if (address != NULL) {
+                msg.msg_name = address;
+                msg.msg_namelen = addrlen;
+            }
+
+            // Add the message
+            struct iovec *iovs = NULL;
+            if (no_of_messages > 0){
+
+                iovs = (struct iovec*) malloc(no_of_messages * sizeof(struct iovec));
+                memset(iovs, 0, no_of_messages * sizeof(struct iovec));
+                msg.msg_iov = iovs;
+                msg.msg_iovlen = no_of_messages;
+
+                for (i=0; i< no_of_messages; i++){
+                    iovs[i].iov_base = messages[i];
+                    iovs[i].iov_len = length_of_messages[i];
+                }
+            }
+
+            // Add the ancillary
+            #ifndef CMSG_SPACE
+                if (control_length > 1){
+                    free(iovs);
+                    return MUL_MSGS_NOT_SUP;
+                }
+            #endif
+            if (control_length > 0){
+
+                //compute the total size of the ancillary
+                //getting the exact amount of space can be tricky and os dependent.
+                size_t total_size_of_ancillary = 0;
+                size_t space;
+                size_t controllen = 0, controllen_last = 0;
+                for (i = 0; i< control_length; i++){
+                    total_size_of_ancillary = no_of_fds[i];
+                    #ifdef CMSG_SPACE
+                        if (!get_CMSG_SPACE(total_size_of_ancillary, &space)) {
+                    #else
+                        if (!get_CMSG_LEN(total_size_of_ancillary, &space)) {
+                    #endif
+                            if (iovs != NULL)
+                                free(iovs);
+                            return ANC_DATA_TOO_LARGE;
+                        }
+                    controllen +=space;
+                    if ((controllen > SOCKLEN_T_LIMIT) || (controllen < controllen_last)) {
+                        if (iovs != NULL)
+                                free(iovs);
+                        return ANC_DATA_TOO_LARGEX;
+                    }
+                    controllen_last = controllen;
+                }
+
+                controlbuf = malloc(controllen);
+                msg.msg_control= controlbuf;
+                msg.msg_controllen = controllen;
+
+                // memset controlbuf to 0 to avoid trash in the ancillary
+                memset(controlbuf, 0, controllen);
+                cmsg = NULL;
+                for (i = 0; i< control_length; i++){
+                    cmsg = (i == 0) ? CMSG_FIRSTHDR(&msg) : CMSG_NXTHDR(&msg, cmsg);
+
+                    cmsg->cmsg_level = (int) levels[i];
+                    cmsg->cmsg_type = (int) types[i];
+                    cmsg->cmsg_len = CMSG_LEN(sizeof(char) * no_of_fds[i]);
+                    memcpy(CMSG_DATA(cmsg), file_descriptors[i], sizeof(char) * no_of_fds[i]);
+                }
+
+
+            }
+            // Add the flags
+            msg.msg_flags = flag;
+
+            // Send the data
+            retval = sendmsg(socket, &msg, flag);
+
+            // free everything that was allocated here, and we would not need in rsocket
+            if (iovs != NULL)
+                free(iovs);
+            if (controlbuf !=NULL)
+               free(controlbuf);
+
+            return retval;
+        }
+
+        // ################################################################################################
+        // Wrappers for CMSG_SPACE and CMSG_LEN
+
+        /*
+            These 2 functions are wrappers over sys/socket.h's CMSG_SPACE and CMSG_LEN.
+            They are identical to CPython's.
+        */
+        #ifdef CMSG_SPACE
+        RPY_EXTERN
+        size_t CMSG_SPACE_wrapper(size_t desired_space){
+            size_t result;
+            if (!get_CMSG_SPACE(desired_space, &result)){
+                return 0;
+            }
+            return result;
+        }
+        #endif
+
+        #ifdef CMSG_LEN
+        RPY_EXTERN
+        size_t CMSG_LEN_wrapper(size_t desired_len){
+            size_t result;
+            if (!get_CMSG_LEN(desired_len, &result)){
+                return 0;
+            }
+            return result;
+        }
+        #endif
+
+        // ################################################################################################
+        // Extra functions that I needed
+
+        /*
+           This function is used to memcpy from a char* at an offset.
+           Could not get rffi.c_memcpy to do it at an offset, so I made my own.
+        */
+        RPY_EXTERN
+        int memcpy_from_CCHARP_at_offset_and_size(char* stringfrom, char** stringto, int offset, int size){
+            *stringto = memcpy(*stringto, stringfrom + offset, size);
+            return 0;
+        }
+
+        /*
+            These functions free memory that was allocated in C (sendmsg or recvmsg) was used in rsocket and now needs cleanup
+        */
+        RPY_EXTERN
+        int free_pointer_to_signedp(int** ptrtofree){
+            free(*ptrtofree);
+            return 0;
+        }
+
+        RPY_EXTERN
+        int free_ptr_to_charp(char** ptrtofree){
+            free(*ptrtofree);
+            return 0;
+        }
+
+    ''',]
+
+    post_include_bits =[ "RPY_EXTERN "
+                         "int sendmsg_implementation(int socket, struct sockaddr* address, socklen_t addrlen, long* length_of_messages, char** messages, int no_of_messages, long* levels, long* types, char** file_descriptors, long* no_of_fds, int control_length, int flag );\n"
+                         "RPY_EXTERN "
+                         "int recvmsg_implementation(int socket_fd, int message_size, int ancillary_size, int flags, struct sockaddr* address, socklen_t* addrlen, long** length_of_messages, char** messages, long* no_of_messages, long* size_of_ancillary, long** levels, long** types, char** file_descr, long** descr_per_ancillary, long* flag);\n"
+                         "static "
+                         "int cmsg_min_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t space);\n"
+                         "static "
+                         "int get_cmsg_data_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *space);\n"
+                         "static "
+                         "int get_cmsg_data_len(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *data_len);\n"
+                         "static "
+                         "int get_CMSG_LEN(size_t length, size_t *result);\n"
+                         "static "
+                         "int get_CMSG_SPACE(size_t length, size_t *result);\n"
+                         "RPY_EXTERN "
+                         "size_t CMSG_LEN_wrapper(size_t desired_len);\n"
+                         "RPY_EXTERN "
+                         "size_t CMSG_SPACE_wrapper(size_t desired_space);\n"
+                         "RPY_EXTERN "
+                         "int memcpy_from_CCHARP_at_offset_and_size(char* stringfrom, char** stringto, int offset, int size);\n"
+                         "RPY_EXTERN "
+                         "int free_pointer_to_signedp(int** ptrtofree);\n"
+                         "RPY_EXTERN "
+                         "int free_ptr_to_charp(char** ptrtofree);\n"
+                         ]
+
+
+    compilation_info = ExternalCompilationInfo(
+                                    includes=includes,
+                                    separate_module_sources=separate_module_sources,
+                                    post_include_bits=post_include_bits,
+                               )
+
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
     CConfig.WSANETWORKEVENTS = platform.Struct(
@@ -387,6 +1035,7 @@
 sockaddr_ptr.TO.become(cConfig.sockaddr)
 addrinfo_ptr.TO.become(cConfig.addrinfo)
 
+
 # fill in missing constants with reasonable defaults
 cConfig.NI_MAXHOST = cConfig.NI_MAXHOST or 1025
 cConfig.NI_MAXSERV = cConfig.NI_MAXSERV or 32
@@ -571,11 +1220,32 @@
 recvfrom = external('recvfrom', [socketfd_type, rffi.VOIDP, size_t,
                            rffi.INT, sockaddr_ptr, socklen_t_ptr], rffi.INT,
                     save_err=SAVE_ERR)
+recvmsg = jit.dont_look_inside(rffi.llexternal("recvmsg_implementation",
+                                               [rffi.INT, rffi.INT, rffi.INT, rffi.INT,sockaddr_ptr, socklen_t_ptr, rffi.SIGNEDPP, rffi.CCHARPP,
+                                                rffi.SIGNEDP,rffi.SIGNEDP, rffi.SIGNEDPP, rffi.SIGNEDPP, rffi.CCHARPP, rffi.SIGNEDPP, rffi.SIGNEDP],
+                                               rffi.INT, save_err=SAVE_ERR,
+                                               compilation_info=compilation_info))
+
+memcpy_from_CCHARP_at_offset = jit.dont_look_inside(rffi.llexternal("memcpy_from_CCHARP_at_offset_and_size",
+                                    [rffi.CCHARP, rffi.CCHARPP,rffi.INT,rffi.INT],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+freeccharp = jit.dont_look_inside(rffi.llexternal("free_ptr_to_charp",
+                                    [rffi.CCHARPP],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+freesignedp = jit.dont_look_inside(rffi.llexternal("free_pointer_to_signedp",
+                                    [rffi.SIGNEDPP],rffi.INT,save_err=SAVE_ERR,compilation_info=compilation_info))
+
 send = external('send', [socketfd_type, rffi.CCHARP, size_t, rffi.INT],
                        ssize_t, save_err=SAVE_ERR)
 sendto = external('sendto', [socketfd_type, rffi.VOIDP, size_t, rffi.INT,
                                     sockaddr_ptr, socklen_t], ssize_t,
                   save_err=SAVE_ERR)
+sendmsg = jit.dont_look_inside(rffi.llexternal("sendmsg_implementation",
+                               [rffi.INT, sockaddr_ptr, socklen_t, rffi.SIGNEDP, rffi.CCHARPP, rffi.INT,
+                                rffi.SIGNEDP, rffi.SIGNEDP, rffi.CCHARPP, rffi.SIGNEDP, rffi.INT, rffi.INT],
+                               rffi.INT, save_err=SAVE_ERR,
+                               compilation_info=compilation_info))
+CMSG_SPACE = jit.dont_look_inside(rffi.llexternal("CMSG_SPACE_wrapper",[size_t], size_t, save_err=SAVE_ERR,compilation_info=compilation_info))
+CMSG_LEN = jit.dont_look_inside(rffi.llexternal("CMSG_LEN_wrapper",[size_t], size_t, save_err=SAVE_ERR,compilation_info=compilation_info))
+
 socketshutdown = external('shutdown', [socketfd_type, rffi.INT], rffi.INT,
                           save_err=SAVE_ERR)
 gethostname = external('gethostname', [rffi.CCHARP, rffi.INT], rffi.INT,
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1312,9 +1312,17 @@
 @replace_os_function('link')
 @specialize.argtype(0, 1)
 def link(oldpath, newpath):
-    oldpath = _as_bytes0(oldpath)
-    newpath = _as_bytes0(newpath)
-    handle_posix_error('link', c_link(oldpath, newpath))
+    if not _WIN32:
+        oldpath = _as_bytes0(oldpath)
+        newpath = _as_bytes0(newpath)
+        handle_posix_error('link', c_link(oldpath, newpath))
+    else:
+        traits = _preferred_traits(oldpath)
+        win32traits = make_win32_traits(traits)
+        oldpath = traits.as_str0(oldpath)
+        newpath = traits.as_str0(newpath)
+        if not win32traits.CreateHardLink(newpath, oldpath, None):
+            raise rwin32.lastSavedWindowsError()
 
 @replace_os_function('symlink')
 @specialize.argtype(0, 1)
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -963,6 +963,126 @@
                 return (read_bytes, address)
             raise self.error_handler()
 
+    @jit.dont_look_inside
+    def recvmsg(self, message_size, ancbufsize = 0, flags = 0):
+        """
+        Receive up to message_size bytes from a message. Also receives ancillary data.
+        Returns the message, ancillary, flag and address of the sender.
+        :param message_size: Maximum size of the message to be received
+        :param ancbufsize:  Maximum size of the ancillary data to be received
+        :param flags: Receive flag. For more details, please check the Unix manual
+        :return: a tuple consisting of the message, the ancillary data, return flag and the address.
+        """
+        if message_size < 0:
+            raise RSocketError("Invalid message size")
+        if ancbufsize < 0:
+            raise RSocketError("invalid ancillary data buffer length")
+
+        self.wait_for_data(False)
+        address, addr_p, addrlen_p  = self._addrbuf()
+        len_of_msgs = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        messages = lltype.malloc(rffi.CCHARPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        messages[0] = lltype.malloc(rffi.CCHARP.TO, message_size,flavor='raw',track_allocation=True,nonmovable=False)
+        rffi.c_memset(messages[0], 0, message_size)
+        no_of_messages  = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        no_of_messages[0] = rffi.cast(rffi.SIGNED, 0)
+        size_of_anc = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        size_of_anc[0] = rffi.cast(rffi.SIGNED,0)
+        levels = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        types = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        file_descr = lltype.malloc(rffi.CCHARPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        descr_per_anc = lltype.malloc(rffi.SIGNEDPP.TO,1,flavor='raw',track_allocation=True,nonmovable=False)
+        retflag = lltype.malloc(rffi.SIGNEDP.TO,1,flavor='raw',track_allocation=True,nonmovable=False )
+        retflag[0] = rffi.cast(rffi.SIGNED,0)
+
+        # a mask for the SIGNEDP's that need to be cast to int. (long default)
+        reply = _c.recvmsg(self.fd, rffi.cast(lltype.Signed,message_size),
+                           rffi.cast(lltype.Signed,ancbufsize),rffi.cast(lltype.Signed,flags),
+                           addr_p, addrlen_p, len_of_msgs, messages, no_of_messages,size_of_anc,
+                           levels, types,file_descr,descr_per_anc,retflag)
+        if reply >= 0:
+            anc_size  = rffi.cast(rffi.SIGNED,size_of_anc[0])
+            returnflag  = rffi.cast(rffi.SIGNED,retflag[0])
+            addrlen = rffi.cast(rffi.SIGNED,addrlen_p[0])
+
+            retmsg = rffi.charpsize2str(messages[0],reply)
+
+            offset = 0
+            list_of_tuples = []
+
+            pre_anc = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw', track_allocation=True, nonmovable=False)
+            for i in range(anc_size):
+                level = rffi.cast(rffi.SIGNED, levels[0][i])
+                type = rffi.cast(rffi.SIGNED, types[0][i])
+                bytes_in_anc = rffi.cast(rffi.SIGNED, descr_per_anc[0][i])
+                pre_anc[0] = lltype.malloc(rffi.CCHARP.TO, bytes_in_anc,flavor='raw',track_allocation=True,nonmovable=False)
+                _c.memcpy_from_CCHARP_at_offset(file_descr[0], pre_anc,rffi.cast(rffi.SIGNED,offset), bytes_in_anc)
+                anc = rffi.charpsize2str(pre_anc[0],bytes_in_anc)
+                tup = (level,type, anc)
+                list_of_tuples.append(tup)
+                offset += bytes_in_anc
+                lltype.free(pre_anc[0], flavor='raw')
+
+            if addrlen:
+                address.addrlen = addrlen
+            else:
+                address.unlock()
+                address = None
+
+            rettup = (retmsg,list_of_tuples,returnflag,address)
+
+            if address is not None:
+                address.unlock()
+            # free underlying complexity first
+            _c.freeccharp(file_descr)
+            _c.freesignedp(len_of_msgs)
+            _c.freesignedp(levels)
+            _c.freesignedp(types)
+            _c.freesignedp(descr_per_anc)
+
+            lltype.free(messages[0], flavor='raw')
+            lltype.free(pre_anc,flavor='raw')
+            lltype.free(messages,flavor='raw')
+            lltype.free(file_descr,flavor='raw')
+            lltype.free(len_of_msgs,flavor='raw')
+            lltype.free(no_of_messages, flavor='raw')
+            lltype.free(size_of_anc, flavor='raw')
+            lltype.free(levels, flavor='raw')
+            lltype.free(types, flavor='raw')
+            lltype.free(descr_per_anc, flavor='raw')
+            lltype.free(retflag, flavor='raw')
+            lltype.free(addrlen_p,flavor='raw')
+
+            return rettup
+        else:
+
+            #in case of failure the underlying complexity has already been freed
+            lltype.free(messages[0], flavor='raw')
+            lltype.free(messages, flavor='raw')
+            lltype.free(file_descr, flavor='raw')
+            lltype.free(len_of_msgs, flavor='raw')
+            lltype.free(no_of_messages, flavor='raw')
+            lltype.free(size_of_anc, flavor='raw')
+            lltype.free(levels, flavor='raw')
+            lltype.free(types, flavor='raw')
+            lltype.free(descr_per_anc, flavor='raw')
+            lltype.free(retflag, flavor='raw')
+            lltype.free(addrlen_p, flavor='raw')
+
+            if address is not None:
+                address.unlock()
+            if _c.geterrno() == _c.EINTR:
+                raise last_error()
+            if (reply == -10000):
+                raise RSocketError("Invalid message size")
+            if (reply == -10001):
+                raise RSocketError("Invalid ancillary data buffer length")
+            if (reply == -10002):
+                raise RSocketError("received malformed or improperly truncated ancillary data")
+            raise last_error()
+
+
+
     def send_raw(self, dataptr, length, flags=0):
         """Send data from a CCHARP buffer."""
         self.wait_for_data(True)
@@ -1009,6 +1129,86 @@
             raise self.error_handler()
         return res
 
+    @jit.dont_look_inside
+    def sendmsg(self, messages, ancillary=None, flags=0, address=None):
+        """
+        Send data and ancillary on a socket. For use of ancillary data, please check the Unix manual.
+        Work on connectionless sockets via the address parameter.
+        :param messages: a message that is a list of strings
+        :param ancillary: data to be sent separate from the message body. Needs to be a list of tuples.
+                            E.g. [(level,type, bytes),...]. Default None.
+        :param flags: the flag to be set for sendmsg. Please check the Unix manual regarding values. Default 0
+        :param address: address of the recepient. Useful for when sending on connectionless sockets. Default None
+        :return: Bytes sent from the message
+        """
+        need_to_free_address = True
+        if address is None:
+            need_to_free_address = False
+            addr = lltype.nullptr(_c.sockaddr)
+            addrlen = 0
+        else:
+            addr = address.lock()
+            addrlen = address.addrlen
+
+        no_of_messages = len(messages)
+        messages_ptr = lltype.malloc(rffi.CCHARPP.TO,no_of_messages+1,flavor='raw',track_allocation=True,nonmovable=False)
+        messages_length_ptr = lltype.malloc(rffi.SIGNEDP.TO,no_of_messages,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        counter = 0
+        for message in messages:
+            messages_ptr[counter] = rffi.str2charp(message)
+            messages_length_ptr[counter] = rffi.cast(rffi.SIGNED, len(message))
+            counter += 1
+        messages_ptr[counter] = lltype.nullptr(rffi.CCHARP.TO)
+        if ancillary is not None:
+            size_of_ancillary = len(ancillary)
+        else:
+            size_of_ancillary = 0
+        levels = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        types = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        desc_per_ancillary = lltype.malloc(rffi.SIGNEDP.TO, size_of_ancillary,flavor='raw',zero=True, track_allocation=True,nonmovable=False)
+        file_descr = lltype.malloc(rffi.CCHARPP.TO, size_of_ancillary,flavor='raw', track_allocation=True,nonmovable=False)
+        if ancillary is not None:
+            counter = 0
+            for level, type, content in ancillary:
+                assert isinstance(type,int)
+                assert isinstance(level, int)
+                levels[counter] = rffi.cast(rffi.SIGNED,level)
+                types[counter] = rffi.cast(rffi.SIGNED,type)
+                desc_per_ancillary[counter] = rffi.cast(rffi.SIGNED, (len(content)))
+                file_descr[counter] = rffi.str2charp(content, track_allocation=True)
+                counter +=1
+        else:
+            size_of_ancillary = 0
+        snd_no_msgs = rffi.cast(rffi.SIGNED, no_of_messages)
+        snd_anc_size =rffi.cast(rffi.SIGNED, size_of_ancillary)
+
+
+        bytes_sent = _c.sendmsg(self.fd, addr, addrlen, messages_length_ptr, messages_ptr, snd_no_msgs,levels,types,file_descr,desc_per_ancillary,snd_anc_size,flags)
+
+
+        if need_to_free_address:
+            address.unlock()
+        for i in range(len(messages)):
+            lltype.free(messages_ptr[i], flavor='raw', track_allocation=True)
+        lltype.free(messages_ptr, flavor='raw', track_allocation=True)
+        lltype.free(messages_length_ptr, flavor='raw', track_allocation=True)
+
+        if size_of_ancillary > 0:
+            for i in range(len(ancillary)):
+                lltype.free(file_descr[i], flavor='raw', track_allocation=True)
+        lltype.free(desc_per_ancillary, flavor='raw', track_allocation=True)
+        lltype.free(types, flavor='raw', track_allocation=True)
+        lltype.free(levels, flavor='raw', track_allocation=True)
+        lltype.free(file_descr, flavor='raw', track_allocation=True)
+
+        self.wait_for_data(True)
+        if (bytes_sent < 0) and (bytes_sent!=-1000) and (bytes_sent!=-1001) and (bytes_sent!=-1002):
+            raise last_error()
+
+        return bytes_sent
+
+
+
     def setblocking(self, block):
         if block:
             timeout = -1.0
@@ -1190,6 +1390,31 @@
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
+if _c._POSIX:
+    def CMSG_LEN( demanded_len):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param demanded_len: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is not memory alligned
+        """
+        if demanded_len < 0:
+            return 0
+        result = _c.CMSG_LEN(demanded_len)
+        return result
+
+    def CMSG_SPACE( demanded_size):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param demanded_size: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is memory alligned
+        """
+        if demanded_size < 0:
+            return 0
+        result = _c.CMSG_SPACE(demanded_size)
+        return result
+
 if _c.WIN32:
     def dup(fd, inheritable=True):
         with lltype.scoped_alloc(_c.WSAPROTOCOL_INFO, zero=True) as info:
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -234,6 +234,12 @@
             rwin32.BOOL,
             save_err=rffi.RFFI_SAVE_LASTERROR)
 
+        CreateHardLink = external(
+            'CreateHardLink' + suffix,
+            [traits.CCHARP, traits.CCHARP, rwin32.LPSECURITY_ATTRIBUTES],
+            rwin32.BOOL,
+            save_err=rffi.RFFI_SAVE_LASTERROR)
+
     return Win32Traits
 
 def make_longlong(high, low):
diff --git a/rpython/rtyper/lltypesystem/rffi.py b/rpython/rtyper/lltypesystem/rffi.py
--- a/rpython/rtyper/lltypesystem/rffi.py
+++ b/rpython/rtyper/lltypesystem/rffi.py
@@ -752,7 +752,8 @@
 
 # Signed, Signed *
 SIGNED = lltype.Signed
-SIGNEDP = lltype.Ptr(lltype.Array(SIGNED, hints={'nolength': True}))
+SIGNEDP = lltype.Ptr(lltype.Array(lltype.Signed, hints={'nolength': True}))
+SIGNEDPP = lltype.Ptr(lltype.Array(SIGNEDP, hints={'nolength': True}))
 
 
 # various type mapping
diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py
--- a/rpython/translator/backendopt/graphanalyze.py
+++ b/rpython/translator/backendopt/graphanalyze.py
@@ -4,6 +4,7 @@
 
 class GraphAnalyzer(object):
     verbose = False
+    explanation = None
 
     def __init__(self, translator):
         self.translator = translator
@@ -73,6 +74,20 @@
     def compute_graph_info(self, graph):
         return None
 
+    def explain_analyze_slowly(self, op):
+        # this is a hack! usually done before a crash
+        self.__init__(self.translator)
+        self.explanation = explanation = []
+        oldverbose = self.verbose
+        self.verbose = True
+        try:
+            self.analyze(op)
+        finally:
+            del self.explanation
+            self.verbose = oldverbose
+        explanation.reverse()
+        return explanation
+
     def analyze(self, op, seen=None, graphinfo=None):
         if op.opname == "direct_call":
             try:
@@ -113,7 +128,11 @@
         return x
 
     def dump_info(self, info):
-        print '[%s] %s' % (self.__class__.__name__, info)
+        st = '[%s] %s' % (self.__class__.__name__, info)
+        if self.explanation is not None:
+            self.explanation.append(st)
+        else:
+            print st
 
     def analyze_direct_call(self, graph, seen=None):
         if seen is None:
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -531,3 +531,37 @@
         typed_effects = self._analyze_graph(t, wa, typed_write)
         typed_effects = self._filter_reads(typed_effects)
         assert typed_effects == direct_effects
+
+    def test_explanation(self):
+        class A(object):
+            def methodname(self):
+                self.x = 1
+                return 1
+            def m(self):
+                raise ValueError
+        class B(A):
+            def methodname(self):
+                return 2
+            def m(self):
+                return 3
+        def fancyname(a):
+            return a.methodname()
+        def m(a):
+            return a.m()
+        def h(flag):
+            if flag:
+                obj = A()
+            else:

From pypy.commits at gmail.com  Wed Sep  6 14:27:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 06 Sep 2017 11:27:31 -0700 (PDT)
Subject: [pypy-commit] pypy pycheck-macros: create non-exporting
 build_type_checkers_flags with fastpath for pto, use everywhere
Message-ID: <59b03e13.88b0df0a.de5c0.3512@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: pycheck-macros
Changeset: r92340:940759a2668e
Date: 2017-09-03 20:56 +0300
http://bitbucket.org/pypy/pypy/changeset/940759a2668e/

Log:	create non-exporting build_type_checkers_flags with fastpath for
	pto, use everywhere

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -753,6 +753,45 @@
 
     return check, check_exact
 
+def build_type_checkers_flags(type_name, cls=None, flagsubstr=None):
+    """
+    Builds two api functions: Py_XxxCheck() and Py_XxxCheckExact()
+    Does not export the functions, assumes they are macros in the *. files
+    check will try a fast path via pto flags
+    """
+    if cls is None:
+        attrname = "w_" + type_name.lower()
+        def get_w_type(space):
+            return getattr(space, attrname)
+    else:
+        def get_w_type(space):
+            return getattr(space, cls)
+    if flagsubstr is None:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % type_name.upper()
+    else:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % flagsubstr
+    check_name = "Py" + type_name + "_Check"
+    tp_flag = globals()[tp_flag_str]
+
+    @specialize.argtype(1)
+    def check(space, pto):
+        from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
+        "Implements the Py_Xxx_Check function"
+        if is_pyobj(pto):
+            return (pto.c_ob_type.c_tp_flags & tp_flag) == tp_flag
+        w_obj_type = space.type(pto)
+        w_type = get_w_type(space)
+        return (space.is_w(w_obj_type, w_type) or
+                space.issubtype_w(w_obj_type, w_type))
+
+    def check_exact(space, w_obj):
+        "Implements the Py_Xxx_CheckExact function"
+        w_obj_type = space.type(w_obj)
+        w_type = get_w_type(space)
+        return space.is_w(w_obj_type, w_type)
+
+    return check, check_exact
+
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
 
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
+    cpython_api, cpython_struct, bootstrap_function, build_type_checkers_flags,
     PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -58,7 +58,7 @@
                    dealloc=bytes_dealloc,
                    realize=bytes_realize)
 
-PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_bytes")
+PyString_Check, PyString_CheckExact = build_type_checkers_flags("String", "w_bytes")
 
 def new_empty_str(space, length):
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -4,7 +4,7 @@
 from pypy.objspace.std.classdict import ClassDictStrategy
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
+    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
     bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
@@ -66,7 +66,7 @@
 def PyDict_New(space):
     return space.newdict()
 
-PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict")
+PyDict_Check, PyDict_CheckExact = build_type_checkers_flags("Dict")
 
 @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, build_type_checkers, bootstrap_function,
+    cpython_api, cpython_struct, build_type_checkers_flags, bootstrap_function,
     PyObject, PyObjectFields, CONST_STRING, CANNOT_FAIL, Py_ssize_t)
 from pypy.module.cpyext.pyobject import (
     make_typedescr, track_reference, from_ref)
@@ -40,7 +40,7 @@
     track_reference(space, obj, w_obj)
     return w_obj
 
-PyInt_Check, PyInt_CheckExact = build_type_checkers("Int")
+PyInt_Check, PyInt_CheckExact = build_type_checkers_flags("Int")
 
 @cpython_api([], lltype.Signed, error=CANNOT_FAIL)
 def PyInt_GetMax(space):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -2,14 +2,14 @@
 from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
-                                    build_type_checkers)
+                                    build_type_checkers_flags)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
 
-PyList_Check, PyList_CheckExact = build_type_checkers("List")
+PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
 def PyList_New(space, len):
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,14 +1,13 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, build_type_checkers, Py_ssize_t,
+    cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
     CONST_STRING, ADDR, CANNOT_FAIL)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
 
-
-PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
+PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long")
 
 @cpython_api([lltype.Signed], PyObject)
 def PyLong_FromLong(space, val):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    build_type_checkers, cpython_api, generic_cpy_call, CANNOT_FAIL,
+    cpython_api, generic_cpy_call, CANNOT_FAIL,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    cpython_api, bootstrap_function, cpython_struct, 
     slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1072,7 +1072,7 @@
 
     def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
-            ("fetchFooType", "NOARGS",
+            ("fetchFooType", "METH_NOARGS",
              """
                 PyObject *o;
                 o = PyObject_New(PyObject, &Foo_Type);
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, build_type_checkers,
+    cpython_api, Py_ssize_t, build_type_checkers_flags,
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
@@ -42,7 +42,7 @@
                    dealloc=tuple_dealloc,
                    realize=tuple_realize)
 
-PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple")
+PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
 def tuple_check_ref(space, ref):
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -13,7 +13,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
     slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers, cts, parse_dir, PyObjectFields, PyTypeObject,
+    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
     PyTypeObjectPtr, Py_TPFLAGS_CHECKTYPES,
     Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_HAVE_GETCHARBUFFER, Py_TPFLAGS_HAVE_INPLACEOPS,
@@ -44,7 +44,7 @@
 
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
-PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
+PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -36,7 +36,7 @@
 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
                                  flavor='raw', zero=True)
 
-PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
+PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers_flags("Unicode")
 
 
 def new_empty_unicode(space, length):

From pypy.commits at gmail.com  Wed Sep  6 12:39:56 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 06 Sep 2017 09:39:56 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: implement hints for int_sub
Message-ID: <59b024dc.24a9df0a.46514.152f@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92337:91c860e7fe64
Date: 2017-09-06 18:14 +0200
http://bitbucket.org/pypy/pypy/changeset/91c860e7fe64/

Log:	implement hints for int_sub

diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -76,6 +76,14 @@
         else:
             self._consider_binop_symm(op, position)
 
+    def consider_int_sub(self, op, position):
+        y = op.getarg(1)
+        if isinstance(y, ConstInt) and rx86.fits_in_32bits(-y.value):
+            pass # nothing to be hinted
+        else:
+            self._consider_binop(op, position)
+
+
     consider_nursery_ptr_increment = consider_int_add
 
     def consider_int_lshift(self, op, position):
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -147,7 +147,7 @@
 
     def test_coalescing(self):
         ops = '''
-        [i0, i1, i2, i3]
+        [i0, i1, i3]
         i7 = int_add(i0, i1)
         i8 = int_add(i7, i3)
         i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
@@ -155,14 +155,28 @@
         guard_true(i10) []
         finish(i9)
         '''
-        self.interpret(ops, [5, 6, 7, 8])
+        self.interpret(ops, [5, 6, 8])
         # coalescing makes sure that i0 (and thus i71) lands in edi
         assert len(self.filter_log_moves()) == 2
 
+    def test_coalescing_sub(self):
+        ops = '''
+        [i0, i1, i3]
+        i7 = int_sub(i0, i1)
+        i8 = int_sub(i7, i3)
+        i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
+        i10 = int_is_true(i9)
+        guard_true(i10) []
+        finish(i9)
+        '''
+        self.interpret(ops, [5, 6, 8])
+        # coalescing makes sure that i7 (and thus i8) lands in edi
+        assert len(self.filter_log_moves()) == 2
+
     def test_coalescing_mul(self):
-        # won't test all operations, but at least check a second one
+        # won't test all symmetric operations, but at least check a second one
         ops = '''
-        [i0, i1, i2, i3]
+        [i0, i1, i3]
         i7 = int_mul(i0, i1)
         i8 = int_mul(i7, i3)
         i9 = call_i(ConstClass(f1ptr), i8, descr=f1_calldescr)
@@ -170,7 +184,7 @@
         guard_true(i10) []
         finish(i9)
         '''
-        self.interpret(ops, [5, 6, 7, 8])
+        self.interpret(ops, [5, 6, 8])
         assert len(self.filter_log_moves()) == 2
 
     def test_lshift(self):

From pypy.commits at gmail.com  Thu Sep  7 16:55:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 13:55:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: Refuse to translate the pyexpat module
 using the UTF-16 version of expat
Message-ID: <59b1b24b.cc981c0a.e9340.0ce6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92344:4589c72f2f47
Date: 2017-09-07 22:54 +0200
http://bitbucket.org/pypy/pypy/changeset/4589c72f2f47/

Log:	Refuse to translate the pyexpat module using the UTF-16 version of
	expat

diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -51,6 +51,13 @@
                  'XML_PARAM_ENTITY_PARSING_ALWAYS']:
         interpleveldefs[name] = 'space.wrap(interp_pyexpat.%s)' % (name,)
 
+    def __init__(self, space, w_name):
+        "NOT_RPYTHON"
+        super(Module, self).__init__(space, w_name)
+        ver = space.unwrap(interp_pyexpat.get_expat_version(space))
+        assert len(ver) >= 5, (
+            "Cannot compile with the wide (UTF-16) version of Expat")
+
     def startup(self, space):
         from pypy.module.pyexpat import interp_pyexpat
         w_ver = interp_pyexpat.get_expat_version(space)

From pypy.commits at gmail.com  Thu Sep  7 16:55:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 13:55:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Kill accidental code duplication
Message-ID: <59b1b248.0bb7df0a.89140.1172@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92343:f1a7daef2ad9
Date: 2017-09-07 22:20 +0200
http://bitbucket.org/pypy/pypy/changeset/f1a7daef2ad9/

Log:	Kill accidental code duplication

diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -119,18 +119,6 @@
         locals()[name] = rffi_platform.ConstantInteger(name)
     for name in xml_model_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
     XML_Parser_SIZE = rffi_platform.SizeOf("XML_Parser")
 
 for k, v in rffi_platform.configure(CConfigure).items():

From pypy.commits at gmail.com  Thu Sep  7 16:55:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 13:55:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <59b1b24d.4e86df0a.82c88.23ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92345:f70ed7c5cc49
Date: 2017-09-07 22:54 +0200
http://bitbucket.org/pypy/pypy/changeset/f70ed7c5cc49/

Log:	merge heads

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1629,12 +1629,15 @@
         # return text_w(w_obj) or None
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    @specialize.argtype(1)
     def bytes_w(self, w_obj):
         """ Takes an application level :py:class:`bytes`
             (on PyPy2 this equals `str`) and returns a rpython byte string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
+    @specialize.argtype(1)
     def text_w(self, w_obj):
         """ PyPy2 takes either a :py:class:`str` and returns a
             rpython byte string, or it takes an :py:class:`unicode`
@@ -1644,6 +1647,7 @@
             On PyPy3 it takes a :py:class:`str` and it will return
             an utf-8 encoded rpython string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1692,6 +1696,7 @@
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
         return chr(value)
 
+    @specialize.argtype(1)
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1704,26 +1709,35 @@
         If allow_conversion=False, w_obj needs to be an app-level int or a
         subclass.
         """
+        assert w_obj is not None
         return w_obj.int_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def int(self, w_obj):
+        assert w_obj is not None
         return w_obj.int(self)
 
+    @specialize.argtype(1)
     def uint_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.uint_w(self)
 
+    @specialize.argtype(1)
     def bigint_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return a rlib.rbigint object and call __long__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.bigint_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def float_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return an interp-level float and call __float__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.float_w(self, allow_conversion)
 
     def realtext_w(self, w_obj):
@@ -1733,7 +1747,9 @@
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
 
+    @specialize.argtype(1)
     def unicode_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.unicode_w(self)
 
     def unicode0_w(self, w_obj):
@@ -1758,7 +1774,9 @@
         # This is here mostly just for gateway.int_unwrapping_space_method().
         return bool(self.int_w(w_obj))
 
+    @specialize.argtype(1)
     def ord(self, w_obj):
+        assert w_obj is not None
         return w_obj.ord(self)
 
     # This is all interface for gateway.py.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
@@ -940,6 +941,11 @@
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
                                                    space.newtext(message)))
+            except rstackovf.StackOverflow as e:
+                rstackovf.check_stack_overflow()
+                failed = True
+                state.set_exception(OperationError(space.w_RuntimeError,
+                         space.newtext("maximum recursion depth exceeded")))
             else:
                 failed = False
 
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -185,6 +185,27 @@
                                          FUNC.RESULT, EffectInfo.MOST_GENERAL)
         return (fnaddr, calldescr)
 
+    def _raise_effect_error(self, op, extraeffect, functype, calling_graph):
+        explanation = []
+        if extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
+            explanation = self.randomeffects_analyzer.explain_analyze_slowly(op)
+        elif extraeffect == EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE:
+            explanation = self.virtualizable_analyzer.explain_analyze_slowly(op)
+        msg = []
+        if explanation:
+            msg = [
+                "_______ ERROR AT BOTTOM ______",
+                "RPython callstack leading to problem:",
+                ]
+            msg.extend(explanation)
+            msg.append("_______ ERROR: ______")
+        msg.append("operation %r" % op)
+        msg.append("in graph %s" % (calling_graph or "<unknown>"))
+        msg.append("this calls a %s function," % (functype, ))
+        msg.append(" but this contradicts other sources (e.g. it can have random"
+                   " effects): EF=%s" % (extraeffect, ))
+        raise Exception("\n".join(msg))
+
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
                      extraeffect=None, extradescr=None,
                      calling_graph=None):
@@ -278,18 +299,13 @@
         # check that the result is really as expected
         if loopinvariant:
             if extraeffect != EffectInfo.EF_LOOPINVARIANT:
-                raise Exception(
-                "operation %r in %s: this calls a _jit_loop_invariant_ function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, calling_graph, extraeffect))
+                self._raise_effect_error(op, extraeffect, "_jit_loop_invariant_", calling_graph)
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
                                    EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
-                raise Exception(
-                "operation %r in %s: this calls an elidable function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, calling_graph, extraeffect))
+
+                self._raise_effect_error(op, extraeffect, "elidable", calling_graph)
             elif RESULT is lltype.Void:
                 raise Exception(
                     "operation %r in %s: this calls an elidable function "
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -299,11 +299,23 @@
     def f4(n, m):
         return compute_hash(str(n) + str(m))
 
+    T = rffi.CArrayPtr(rffi.TIME_T)
+    external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True)
+
+    def effect():
+        return external(lltype.nullptr(T.TO))
+
+    @jit.elidable
+    def f5(n, m):
+        effect()
+        return 1
+
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
         d = f4(n, m)
+        f5(n, m)
         enable_siphash24()
         return a + len(b) + c + d
 
@@ -323,6 +335,14 @@
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.extraeffect == expected
 
+    call_op = f_graph.startblock.operations[4]
+    assert call_op.opname == 'direct_call'
+    excinfo = py.test.raises(Exception, cc.getcalldescr, call_op)
+    lines = excinfo.value.args[0].splitlines()
+    assert "f5" in lines[2]
+    assert "effect" in lines[3]
+    assert "random effects" in lines[-1]
+
 def test_raise_elidable_no_result():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
     l = []
diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py
--- a/rpython/translator/backendopt/graphanalyze.py
+++ b/rpython/translator/backendopt/graphanalyze.py
@@ -4,6 +4,7 @@
 
 class GraphAnalyzer(object):
     verbose = False
+    explanation = None
 
     def __init__(self, translator):
         self.translator = translator
@@ -73,6 +74,20 @@
     def compute_graph_info(self, graph):
         return None
 
+    def explain_analyze_slowly(self, op):
+        # this is a hack! usually done before a crash
+        self.__init__(self.translator)
+        self.explanation = explanation = []
+        oldverbose = self.verbose
+        self.verbose = True
+        try:
+            self.analyze(op)
+        finally:
+            del self.explanation
+            self.verbose = oldverbose
+        explanation.reverse()
+        return explanation
+
     def analyze(self, op, seen=None, graphinfo=None):
         if op.opname == "direct_call":
             try:
@@ -113,7 +128,11 @@
         return x
 
     def dump_info(self, info):
-        print '[%s] %s' % (self.__class__.__name__, info)
+        st = '[%s] %s' % (self.__class__.__name__, info)
+        if self.explanation is not None:
+            self.explanation.append(st)
+        else:
+            print st
 
     def analyze_direct_call(self, graph, seen=None):
         if seen is None:
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -531,3 +531,37 @@
         typed_effects = self._analyze_graph(t, wa, typed_write)
         typed_effects = self._filter_reads(typed_effects)
         assert typed_effects == direct_effects
+
+    def test_explanation(self):
+        class A(object):
+            def methodname(self):
+                self.x = 1
+                return 1
+            def m(self):
+                raise ValueError
+        class B(A):
+            def methodname(self):
+                return 2
+            def m(self):
+                return 3
+        def fancyname(a):
+            return a.methodname()
+        def m(a):
+            return a.m()
+        def h(flag):
+            if flag:
+                obj = A()
+            else:
+                obj = B()
+            fancyname(obj)
+            m(obj)
+
+        t, wa = self.translate(h, [int])
+        hgraph = graphof(t, h)
+        # fiiiish :-(
+        block = hgraph.startblock.exits[0].target.exits[0].target
+        op_call_fancyname = block.operations[0]
+
+        explanation = wa.explain_analyze_slowly(op_call_fancyname)
+        assert "fancyname" in explanation[0]
+        assert "methodname" in explanation[1]

From pypy.commits at gmail.com  Thu Sep  7 17:05:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 14:05:14 -0700 (PDT)
Subject: [pypy-commit] buildbot cleanup-hg-bookmarks: Tentative Windows
 version
Message-ID: <59b1b48a.86bc1c0a.50aec.0c2c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cleanup-hg-bookmarks
Changeset: r1024:da40d3078ef5
Date: 2017-09-07 23:05 +0200
http://bitbucket.org/pypy/buildbot/changeset/da40d3078ef5/

Log:	Tentative Windows version

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -346,7 +346,8 @@
         # the local checkout.  So, manually clean it up.
         factory.addStep(ShellCmd(
             description="cleanup bookmarks",
-            command=["rm", "-f", ".hg/bookmarks"],
+            command=["rm", "-f", ".hg/bookmarks"] if platform != 'win32'
+               else ["if exist .hg/bookmarks del .hg/bookmarks"],
             workdir=workdir,
             haltOnFailure=False,
         ))

From pypy.commits at gmail.com  Thu Sep  7 17:09:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 14:09:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Blind fix for Windows
Message-ID: <59b1b56c.f781df0a.c1dd0.17c4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92346:4484e0986b55
Date: 2017-09-07 23:08 +0200
http://bitbucket.org/pypy/pypy/changeset/4484e0986b55/

Log:	Blind fix for Windows

diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -985,11 +985,14 @@
                          ]
 
 
-    compilation_info = ExternalCompilationInfo(
+    compilation_info = eci.merge(ExternalCompilationInfo(
                                     includes=includes,
                                     separate_module_sources=separate_module_sources,
                                     post_include_bits=post_include_bits,
-                               )
+                               ))
+else:
+    compilation_info = eci
+
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)

From pypy.commits at gmail.com  Thu Sep  7 19:11:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 07 Sep 2017 16:11:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add missing import.  No clue why,
 it is not necessary to run the tests???
Message-ID: <59b1d22f.c59c1c0a.9a2d6.4373@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92347:a38744e20a86
Date: 2017-09-08 01:11 +0200
http://bitbucket.org/pypy/pypy/changeset/a38744e20a86/

Log:	Add missing import. No clue why, it is not necessary to run the
	tests???

diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -53,6 +53,7 @@
 
     def __init__(self, space, w_name):
         "NOT_RPYTHON"
+        from pypy.module.pyexpat import interp_pyexpat
         super(Module, self).__init__(space, w_name)
         ver = space.unwrap(interp_pyexpat.get_expat_version(space))
         assert len(ver) >= 5, (

From pypy.commits at gmail.com  Fri Sep  8 03:07:35 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 00:07:35 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: adds hints for target
 registers of jumps as well
Message-ID: <59b241b7.07c41c0a.79a7b.5a91@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92348:baece8cd15d9
Date: 2017-09-07 14:17 +0200
http://bitbucket.org/pypy/pypy/changeset/baece8cd15d9/

Log:	adds hints for target registers of jumps as well

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -156,6 +156,7 @@
         # to be read/used by the assembler too
         self.jump_target_descr = None
         self.final_jump_op = None
+        self.final_jump_op_position = -1
 
     def _prepare(self, inputargs, operations, allgcrefs):
         from rpython.jit.backend.x86.reghint import X86RegisterHints
@@ -1318,19 +1319,20 @@
         if op.getopnum() != rop.JUMP:
             return
         self.final_jump_op = op
+        self.final_jump_op_position = len(operations) - 1
         descr = op.getdescr()
         assert isinstance(descr, TargetToken)
         if descr._ll_loop_code != 0:
             # if the target LABEL was already compiled, i.e. if it belongs
             # to some already-compiled piece of code
-            self._compute_hint_frame_locations_from_descr(descr)
+            self._compute_hint_locations_from_descr(descr)
         #else:
         #   The loop ends in a JUMP going back to a LABEL in the same loop.
         #   We cannot fill 'hint_frame_pos' immediately, but we can
         #   wait until the corresponding consider_label() to know where the
         #   we would like the boxes to be after the jump.
 
-    def _compute_hint_frame_locations_from_descr(self, descr):
+    def _compute_hint_locations_from_descr(self, descr):
         arglocs = descr._x86_arglocs
         jump_op = self.final_jump_op
         assert len(arglocs) == jump_op.numargs()
@@ -1340,6 +1342,11 @@
                 loc = arglocs[i]
                 if isinstance(loc, FrameLoc):
                     self.fm.hint_frame_pos[box] = self.fm.get_loc_index(loc)
+                else:
+                    assert isinstance(loc, RegLoc)
+                    self.longevity.fixed_register(
+                            self.final_jump_op_position,
+                            loc, box)
 
     def consider_jump(self, op):
         assembler = self.assembler
@@ -1447,7 +1454,7 @@
         # the hints about the expected position of the spilled variables.
         jump_op = self.final_jump_op
         if jump_op is not None and jump_op.getdescr() is descr:
-            self._compute_hint_frame_locations_from_descr(descr)
+            self._compute_hint_locations_from_descr(descr)
 
     def consider_guard_not_forced_2(self, op):
         self.rm.before_call(op.getfailargs(), save_all_regs=True)
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -51,6 +51,13 @@
     def malloc_cond(self, nursery_free_adr, nursery_top_adr, size, gcmap):
         self._log("malloc_cond", size, "ecx") # always uses edx and ecx
 
+    def label(self):
+        self._log("label")
+        return Assembler386.label(self)
+
+    def closing_jump(self, jump_target_descr):
+        self._log("jump")
+        return Assembler386.closing_jump(self, jump_target_descr)
 
 class TestCheckRegistersExplicitly(test_regalloc_integration.BaseTestRegalloc):
     def setup_class(cls):
@@ -254,6 +261,41 @@
         # edx for any of the integer results
         assert len(self.filter_log_moves()) == 2
 
+    def test_jump_hinting(self):
+        ops = '''
+        [i0]
+        i1 = int_add(i0, 1)
+        i10 = int_add(i1, 1)
+        i2 = int_add(i1, 1)
+        i3 = int_lt(i2, 20)
+        guard_true(i3) [i1, i10]
+        label(i2, descr=targettoken)
+        i4 = int_add(i2, 1)
+        i11 = int_add(i4, 1)
+        i5 = int_add(i4, 1)
+        i6 = int_lt(i5, 20)
+        guard_true(i6) [i4, i11]
+        jump(i5, descr=targettoken)
+        '''
+        self.interpret(ops, [0], run=False)
+        assert len(self.filter_log_moves()) == 1
+
+    @pytest.mark.skip("later")
+    def test_jump_different_args2(self):
+        ops = '''
+        [i0, i4, i6]
+        i1 = int_add(i0, i6)
+        i2 = int_lt(i1, 20)
+        guard_true(i2) [i1]
+        label(i4, i1, i6, descr=targettoken)
+        i3 = int_add(i4, i6)
+        i7 = int_lt(i3, 20)
+        guard_true(i7) [i3]
+        jump(i1, i3, i6, descr=targettoken)
+        '''
+        self.interpret(ops, [0], run=False)
+
+
     def test_flowcontext(self):
         # real index manipulation for a slicing operation done when translating
         # on top of pypy

From pypy.commits at gmail.com  Fri Sep  8 03:07:37 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 00:07:37 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: make allocation decisions
 in the LEA case a bit more flexible by freeing the
Message-ID: <59b241b9.84901c0a.2bcba.8dd1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92349:7a3cd6204340
Date: 2017-09-07 14:32 +0200
http://bitbucket.org/pypy/pypy/changeset/7a3cd6204340/

Log:	make allocation decisions in the LEA case a bit more flexible by
	freeing the argument

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -533,7 +533,11 @@
         self.perform(op, [loc, argloc], loc)
 
     def _consider_lea(self, op):
-        loc = self.make_sure_var_in_reg(op.getarg(0))
+        x = op.getarg(0)
+        loc = self.make_sure_var_in_reg(x)
+        # make it possible to have argloc be == loc if x dies
+        # (then LEA will not be used, but that's fine anyway)
+        self.possibly_free_var(x)
         argloc = self.loc(op.getarg(1))
         resloc = self.force_allocate_reg(op)
         self.perform(op, [loc, argloc], resloc)
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -262,6 +262,7 @@
         assert len(self.filter_log_moves()) == 2
 
     def test_jump_hinting(self):
+        self.targettoken._ll_loop_code = 0
         ops = '''
         [i0]
         i1 = int_add(i0, 1)
@@ -280,6 +281,22 @@
         self.interpret(ops, [0], run=False)
         assert len(self.filter_log_moves()) == 1
 
+    def test_jump_hinting_int_add(self):
+        self.targettoken._ll_loop_code = 0
+        ops = '''
+        [i0]
+        i1 = int_add(i0, 1)
+        i3 = int_lt(i1, 20)
+        guard_true(i3) [i1]
+        label(i1, descr=targettoken)
+        i4 = int_add(i1, 1)
+        i6 = int_lt(i4, 20)
+        guard_true(i6) [i4]
+        jump(i4, descr=targettoken)
+        '''
+        self.interpret(ops, [0], run=False)
+        assert len(self.filter_log_moves()) == 1
+
     @pytest.mark.skip("later")
     def test_jump_different_args2(self):
         ops = '''

From pypy.commits at gmail.com  Fri Sep  8 03:07:39 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 00:07:39 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: test and fix for corner case
Message-ID: <59b241bb.cd5e1c0a.73fbf.78a6@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92350:58247cb00737
Date: 2017-09-07 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/58247cb00737/

Log:	test and fix for corner case

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -841,6 +841,8 @@
         l = self.real_usages
         low = 0
         high = len(l)
+        if position >= l[-1]:
+            return -1
         while low < high:
             mid = low + (high - low) // 2 # no overflow ;-)
             if position < l[mid]:
diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -140,6 +140,8 @@
         assert next in lt.real_usages
         assert next > i
         assert lt.real_usages[lt.real_usages.index(next) - 1] <= i
+    assert lt.next_real_usage(100) == -1
+    assert lt.next_real_usage(101) == -1
 
 def test_fixed_position():
     b0, b1, b2 = newboxes(0, 0, 0)

From pypy.commits at gmail.com  Fri Sep  8 03:07:41 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 00:07:41 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: this one passes now
Message-ID: <59b241bd.318fdf0a.feee2.a986@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92351:7636f0eb120c
Date: 2017-09-07 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/7636f0eb120c/

Log:	this one passes now

diff --git a/rpython/jit/backend/llsupport/test/test_regalloc.py b/rpython/jit/backend/llsupport/test/test_regalloc.py
--- a/rpython/jit/backend/llsupport/test/test_regalloc.py
+++ b/rpython/jit/backend/llsupport/test/test_regalloc.py
@@ -1205,11 +1205,6 @@
         self.regalloc = regalloc
         return regalloc.fake_allocate(loop)
 
-    def _consider_binop(self, op):
-        loc, argloc = self._consider_binop_part(op)
-        self.perform(op, [loc, argloc], loc)
-
-
     def test_simple(self):
         ops = '''
         [i0]
@@ -1306,7 +1301,6 @@
             ('guard_true', r8, [])
         ]
 
-    @py.test.mark.skip("messy - later")
     def test_coalescing_first_var_already_in_different_reg(self):
         ops = '''
         [i0]
@@ -1321,7 +1315,7 @@
             ('int_mul', r1, [2]),
             ('int_add', r1, [1]),
             ('call_i', r0, [r1]),
-            ('guard_false', r0, [])
+            ('guard_false', r0, [r5])
         ]
 
     def test_call_spill_furthest_use(self):

From pypy.commits at gmail.com  Fri Sep  8 03:25:47 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 00:25:47 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: fix for duplicate jump args
Message-ID: <59b245fb.499ddf0a.5807c.4387@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92352:d4ec0e2656cd
Date: 2017-09-08 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/d4ec0e2656cd/

Log:	fix for duplicate jump args

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -1340,6 +1340,7 @@
         arglocs = descr._x86_arglocs
         jump_op = self.final_jump_op
         assert len(arglocs) == jump_op.numargs()
+        hinted = []
         for i in range(jump_op.numargs()):
             box = jump_op.getarg(i)
             if not isinstance(box, Const):
@@ -1347,10 +1348,12 @@
                 if isinstance(loc, FrameLoc):
                     self.fm.hint_frame_pos[box] = self.fm.get_loc_index(loc)
                 else:
-                    assert isinstance(loc, RegLoc)
-                    self.longevity.fixed_register(
-                            self.final_jump_op_position,
-                            loc, box)
+                    if box not in hinted:
+                        hinted.append(box)
+                        assert isinstance(loc, RegLoc)
+                        self.longevity.fixed_register(
+                                self.final_jump_op_position,
+                                loc, box)
 
     def consider_jump(self, op):
         assembler = self.assembler
diff --git a/rpython/jit/backend/x86/test/test_regalloc.py b/rpython/jit/backend/x86/test/test_regalloc.py
--- a/rpython/jit/backend/x86/test/test_regalloc.py
+++ b/rpython/jit/backend/x86/test/test_regalloc.py
@@ -52,11 +52,11 @@
         self._log("malloc_cond", size, "ecx") # always uses edx and ecx
 
     def label(self):
-        self._log("label")
+        self._log("label", self._regalloc.final_jump_op.getdescr()._x86_arglocs)
         return Assembler386.label(self)
 
     def closing_jump(self, jump_target_descr):
-        self._log("jump")
+        self._log("jump", self._regalloc.final_jump_op.getdescr()._x86_arglocs)
         return Assembler386.closing_jump(self, jump_target_descr)
 
 class TestCheckRegistersExplicitly(test_regalloc_integration.BaseTestRegalloc):
@@ -243,7 +243,6 @@
     def test_malloc(self, monkeypatch):
         ops = '''
         [i0]
-        label(i0, descr=targettoken)
         i1 = int_add(i0, 1) # this is using ecx or edx because it fits
         i6 = int_add(i0, 6) # this is using ecx or edx because it fits
         i2 = int_add(i6, i1)
@@ -281,6 +280,27 @@
         self.interpret(ops, [0], run=False)
         assert len(self.filter_log_moves()) == 1
 
+    def test_jump_hinting_duplicate(self):
+        self.targettoken._ll_loop_code = 0
+        ops = '''
+        [i0]
+        i1 = int_add(i0, 1)
+        i10 = int_add(i1, 1)
+        i2 = int_add(i1, 1)
+        i3 = int_lt(i2, 20)
+        guard_true(i3) [i1, i10]
+        label(i2, i10, descr=targettoken)
+        i4 = int_add(i2, 1)
+        i11 = int_add(i4, i10)
+        i5 = int_add(i4, 1)
+        i6 = int_lt(i5, 20)
+        guard_true(i6) [i4, i11]
+        jump(i5, i5, descr=targettoken)
+        '''
+        self.interpret(ops, [0], run=False)
+        assert len(self.filter_log_moves()) == 3
+
+
     def test_jump_hinting_int_add(self):
         self.targettoken._ll_loop_code = 0
         ops = '''

From pypy.commits at gmail.com  Fri Sep  8 10:29:57 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 08 Sep 2017 07:29:57 -0700 (PDT)
Subject: [pypy-commit] pypy pycheck-macros: document,
 close branch to be merged
Message-ID: <59b2a965.aa9adf0a.a81d5.d561@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: pycheck-macros
Changeset: r92353:d9da470c2a62
Date: 2017-09-08 17:24 +0300
http://bitbucket.org/pypy/pypy/changeset/d9da470c2a62/

Log:	document, close branch to be merged

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -77,3 +77,7 @@
 .. branch: pypy_swappedbytes
 
 Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.

From pypy.commits at gmail.com  Fri Sep  8 10:30:01 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 08 Sep 2017 07:30:01 -0700 (PDT)
Subject: [pypy-commit] pypy default: make quoting error message more
 informative and cpython-compatible
Message-ID: <59b2a969.cb451c0a.76c9e.a1c7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92355:c4c44cc4a4de
Date: 2017-09-08 15:51 +0300
http://bitbucket.org/pypy/pypy/changeset/c4c44cc4a4de/

Log:	make quoting error message more informative and cpython-compatible

diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -29,10 +29,15 @@
         return default
     return space.is_true(w_src)
 
-def _get_int(space, w_src, default):
+def _get_int(space, w_src, default, attrname):
     if w_src is None:
         return default
-    return space.int_w(w_src)
+    try:
+        return space.int_w(w_src)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
+        raise
 
 def _get_str(space, w_src, default, attrname):
     if w_src is None:
@@ -100,7 +105,7 @@
     dialect.escapechar = _get_char(space, w_escapechar, '\0', 'escapechar')
     dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
     dialect.quotechar = _get_char(space, w_quotechar, '"', 'quotechar')
-    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL)
+    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
 
diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py
--- a/pypy/module/_csv/test/test_dialect.py
+++ b/pypy/module/_csv/test/test_dialect.py
@@ -65,7 +65,8 @@
             name = attempt[0]
             for value in attempt[1:]:
                 kwargs = {name: value}
-                raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                exc_info = raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                assert name in exc_info.value.args[0]
 
         exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4)
         assert exc_info.value.args[0] == '"lineterminator" must be a string'

From pypy.commits at gmail.com  Fri Sep  8 10:29:59 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 08 Sep 2017 07:29:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge pycheck-macros,
 which avoids cpyext for many Py*_Check function calls
Message-ID: <59b2a967.cd5e1c0a.41c10.2623@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92354:2782c2c4ec0b
Date: 2017-09-08 17:25 +0300
http://bitbucket.org/pypy/pypy/changeset/2782c2c4ec0b/

Log:	merge pycheck-macros, which avoids cpyext for many Py*_Check
	function calls

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -77,3 +77,7 @@
 .. branch: pypy_swappedbytes
 
 Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -129,6 +129,11 @@
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()
+
+for name in ('INT', 'LONG', 'LIST', 'TUPLE', 'UNICODE', 'DICT', 'BASE_EXC',
+             'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3
+    constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name)
+
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
 globals().update(rffi_platform.configure(CConfig_constants))
@@ -749,6 +754,45 @@
 
     return check, check_exact
 
+def build_type_checkers_flags(type_name, cls=None, flagsubstr=None):
+    """
+    Builds two api functions: Py_XxxCheck() and Py_XxxCheckExact()
+    Does not export the functions, assumes they are macros in the *. files
+    check will try a fast path via pto flags
+    """
+    if cls is None:
+        attrname = "w_" + type_name.lower()
+        def get_w_type(space):
+            return getattr(space, attrname)
+    else:
+        def get_w_type(space):
+            return getattr(space, cls)
+    if flagsubstr is None:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % type_name.upper()
+    else:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % flagsubstr
+    check_name = "Py" + type_name + "_Check"
+    tp_flag = globals()[tp_flag_str]
+
+    @specialize.argtype(1)
+    def check(space, pto):
+        from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
+        "Implements the Py_Xxx_Check function"
+        if is_pyobj(pto):
+            return (pto.c_ob_type.c_tp_flags & tp_flag) == tp_flag
+        w_obj_type = space.type(pto)
+        w_type = get_w_type(space)
+        return (space.is_w(w_obj_type, w_type) or
+                space.issubtype_w(w_obj_type, w_type))
+
+    def check_exact(space, w_obj):
+        "Implements the Py_Xxx_CheckExact function"
+        w_obj_type = space.type(w_obj)
+        w_type = get_w_type(space)
+        return space.is_w(w_obj_type, w_type)
+
+    return check, check_exact
+
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
 
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
+    cpython_api, cpython_struct, bootstrap_function, build_type_checkers_flags,
     PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -58,7 +58,7 @@
                    dealloc=bytes_dealloc,
                    realize=bytes_realize)
 
-PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_bytes")
+PyString_Check, PyString_CheckExact = build_type_checkers_flags("String", "w_bytes")
 
 def new_empty_str(space, length):
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -4,7 +4,7 @@
 from pypy.objspace.std.classdict import ClassDictStrategy
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
+    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
     bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
@@ -66,7 +66,7 @@
 def PyDict_New(space):
     return space.newdict()
 
-PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict")
+PyDict_Check, PyDict_CheckExact = build_type_checkers_flags("Dict")
 
 @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
diff --git a/pypy/module/cpyext/include/dictobject.h b/pypy/module/cpyext/include/dictobject.h
--- a/pypy/module/cpyext/include/dictobject.h
+++ b/pypy/module/cpyext/include/dictobject.h
@@ -12,6 +12,10 @@
     PyObject *_tmpkeys; /* a private place to put keys during PyDict_Next */
 } PyDictObject;
 
+#define PyDict_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_DICT_SUBCLASS)
+#define PyDict_CheckExact(op) ((op)->ob_type == &PyDict_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/intobject.h b/pypy/module/cpyext/include/intobject.h
--- a/pypy/module/cpyext/include/intobject.h
+++ b/pypy/module/cpyext/include/intobject.h
@@ -12,6 +12,10 @@
     long ob_ival;
 } PyIntObject;
 
+#define PyInt_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_INT_SUBCLASS)
+#define PyInt_CheckExact(op) ((op)->ob_type == &PyInt_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,4 @@
 /* empty */
+#define PyList_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS)
+#define PyList_CheckExact(op) ((op)->ob_type == &PyList_Type)
diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h
--- a/pypy/module/cpyext/include/longobject.h
+++ b/pypy/module/cpyext/include/longobject.h
@@ -14,6 +14,9 @@
 
 #define PyOS_strtoul strtoul
 #define PyOS_strtol strtoul
+#define PyLong_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS)
+#define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -236,6 +236,11 @@
 #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
 
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
+#define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
+
+#define PyType_Check(op) \
+    PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
+#define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type)
 
 /* objimpl.h ----------------------------------------------*/
 #define PyObject_New(type, typeobj) \
diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h
--- a/pypy/module/cpyext/include/pyerrors.h
+++ b/pypy/module/cpyext/include/pyerrors.h
@@ -9,7 +9,7 @@
 
 #define PyExceptionClass_Check(x)                                       \
     (PyClass_Check((x)) || (PyType_Check((x)) &&                        \
-      PyObject_IsSubclass((x), PyExc_BaseException)))
+      PyType_FastSubclass((PyTypeObject*)(x), Py_TPFLAGS_BASE_EXC_SUBCLASS)))
 
 PyAPI_FUNC(PyObject *) PyErr_NewException(const char *name, PyObject *base, PyObject *dict);
 PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict);
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -61,6 +61,10 @@
 PyAPI_FUNC(PyObject *) PyString_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyString_FromFormat(const char *format, ...);
 
+#define PyString_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_STRING_SUBCLASS)
+#define PyString_CheckExact(op) ((op)->ob_type == &PyString_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -26,6 +26,9 @@
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+#define PyTuple_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_TUPLE_SUBCLASS)
+#define PyTuple_CheckExact(op) ((op)->ob_type == &PyTuple_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,10 @@
 
 #include "cpyext_unicodeobject.h"
 
+#define PyUnicode_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS)
+#define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, build_type_checkers, bootstrap_function,
+    cpython_api, cpython_struct, build_type_checkers_flags, bootstrap_function,
     PyObject, PyObjectFields, CONST_STRING, CANNOT_FAIL, Py_ssize_t)
 from pypy.module.cpyext.pyobject import (
     make_typedescr, track_reference, from_ref)
@@ -40,7 +40,7 @@
     track_reference(space, obj, w_obj)
     return w_obj
 
-PyInt_Check, PyInt_CheckExact = build_type_checkers("Int")
+PyInt_Check, PyInt_CheckExact = build_type_checkers_flags("Int")
 
 @cpython_api([], lltype.Signed, error=CANNOT_FAIL)
 def PyInt_GetMax(space):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -2,14 +2,14 @@
 from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
-                                    build_type_checkers)
+                                    build_type_checkers_flags)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
 
-PyList_Check, PyList_CheckExact = build_type_checkers("List")
+PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
 def PyList_New(space, len):
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,14 +1,13 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, build_type_checkers, Py_ssize_t,
+    cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
     CONST_STRING, ADDR, CANNOT_FAIL)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
 
-
-PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
+PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long")
 
 @cpython_api([lltype.Signed], PyObject)
 def PyLong_FromLong(space, val):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    build_type_checkers, cpython_api, generic_cpy_call, CANNOT_FAIL,
+    cpython_api, generic_cpy_call, CANNOT_FAIL,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    cpython_api, bootstrap_function, cpython_struct, 
     slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1072,7 +1072,7 @@
 
     def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
-            ("fetchFooType", "METH_VARARGS",
+            ("fetchFooType", "METH_NOARGS",
              """
                 PyObject *o;
                 o = PyObject_New(PyObject, &Foo_Type);
@@ -1090,7 +1090,7 @@
                 Py_DECREF(e);
                 return o;
              """),
-            ("getCounter", "METH_VARARGS",
+            ("getCounter", "METH_NOARGS",
              """
                 return PyInt_FromLong(foo_counter);
              """)], prologue="""
@@ -1377,3 +1377,53 @@
         # this is equivalent to
         from collections import Hashable
         assert not isinstance(obj, Hashable)
+
+
+class AppTestFlags(AppTestCpythonExtensionBase):
+    def test_has_subclass_flag(self):
+        module = self.import_extension('foo', [
+           ("test_flags", "METH_VARARGS",
+            '''
+                long in_flag, my_flag;
+                PyObject * obj;
+                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                    return NULL;
+                if (!PyType_Check(obj))
+                {
+                    PyErr_SetString(PyExc_ValueError, "input must be type");
+                    return NULL;
+                }
+                my_flag = ((PyTypeObject*)obj)->tp_flags;
+                if ((my_flag & in_flag) != in_flag)
+                    return PyLong_FromLong(-1);
+                if (!PyType_CheckExact(obj)) {
+                    if ((my_flag & Py_TPFLAGS_TYPE_SUBCLASS) == Py_TPFLAGS_TYPE_SUBCLASS)
+                        return PyLong_FromLong(-2);
+                }
+                return PyLong_FromLong(0);
+            '''),])
+        # copied from object.h
+        Py_TPFLAGS_INT_SUBCLASS       = (1L<<23) # goes away on py3
+        Py_TPFLAGS_LONG_SUBCLASS      = (1L<<24)
+        Py_TPFLAGS_LIST_SUBCLASS      = (1L<<25)
+        Py_TPFLAGS_TUPLE_SUBCLASS     = (1L<<26)
+        Py_TPFLAGS_STRING_SUBCLASS    = (1L<<27) # rename to BYTES on py3
+        Py_TPFLAGS_UNICODE_SUBCLASS   = (1L<<28)
+        Py_TPFLAGS_DICT_SUBCLASS      = (1L<<29)
+        Py_TPFLAGS_BASE_EXC_SUBCLASS  = (1L<<30)
+        Py_TPFLAGS_TYPE_SUBCLASS      = (1L<<31)
+        for t,f in ((long, Py_TPFLAGS_LONG_SUBCLASS),
+                    (int, Py_TPFLAGS_INT_SUBCLASS),
+                    (list, Py_TPFLAGS_LIST_SUBCLASS),
+                    (tuple, Py_TPFLAGS_TUPLE_SUBCLASS),
+                    (str, Py_TPFLAGS_STRING_SUBCLASS),
+                    (unicode, Py_TPFLAGS_UNICODE_SUBCLASS),
+                    (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS),
+                    (type, Py_TPFLAGS_TYPE_SUBCLASS),
+                   ):
+            assert module.test_flags(t, f) == 0
+        class MyList(list):
+            pass
+        assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0
+
+         
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, build_type_checkers,
+    cpython_api, Py_ssize_t, build_type_checkers_flags,
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
@@ -42,7 +42,7 @@
                    dealloc=tuple_dealloc,
                    realize=tuple_realize)
 
-PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple")
+PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
 def tuple_check_ref(space, ref):
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -12,12 +12,17 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
-    Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
-    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
-    Py_TPFLAGS_HAVE_INPLACEOPS, cts, parse_dir)
+    slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
+    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
+    PyTypeObjectPtr, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, Py_TPFLAGS_HAVE_INPLACEOPS,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
+    Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
+    Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
+    Py_TPFLAGS_TYPE_SUBCLASS,
+    Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
+    )
 from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -39,7 +44,7 @@
 
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
-PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
+PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
@@ -428,7 +433,7 @@
     dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space),
                                           from_ref(space, pyo), None)
 
-def inherit_special(space, pto, base_pto):
+def inherit_special(space, pto, w_obj, base_pto):
     # XXX missing: copy basicsize and flags in a magical way
     # (minimally, if tp_basicsize is zero or too low, we copy it from the base)
     if pto.c_tp_basicsize < base_pto.c_tp_basicsize:
@@ -438,6 +443,26 @@
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
 
+    #/* Setup fast subclass flags */
+    if space.issubtype_w(w_obj, space.w_Exception):
+        pto.c_tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_type):
+        pto.c_tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_int): # remove on py3
+        pto.c_tp_flags |= Py_TPFLAGS_INT_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_long):
+        pto.c_tp_flags |= Py_TPFLAGS_LONG_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_bytes): 
+        pto.c_tp_flags |= Py_TPFLAGS_STRING_SUBCLASS # STRING->BYTES on py3
+    elif space.issubtype_w(w_obj, space.w_unicode):
+        pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_tuple):
+        pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_list):
+        pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_dict):
+        pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS
+
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
         raise DescrMismatch()
@@ -939,7 +964,7 @@
     pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w))
     base = pto.c_tp_base
     if base:
-        inherit_special(space, pto, base)
+        inherit_special(space, pto, w_obj, base)
     for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)):
         if isinstance(w_base, W_TypeObject):
             inherit_slots(space, pto, w_base)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -36,7 +36,7 @@
 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
                                  flavor='raw', zero=True)
 
-PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
+PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers_flags("Unicode")
 
 
 def new_empty_unicode(space, length):

From pypy.commits at gmail.com  Fri Sep  8 11:23:30 2017
From: pypy.commits at gmail.com (asottile)
Date: Fri, 08 Sep 2017 08:23:30 -0700 (PDT)
Subject: [pypy-commit] pypy py_ssize_t: Explicitly use Py_ssize_t as the
 Signed type in pypy c-api.
Message-ID: <59b2b5f2.e6361c0a.44c83.97e9@mx.google.com>

Author: Anthony Sottile <asottile at umich.edu>
Branch: py_ssize_t
Changeset: r92356:46c6c05fde4b
Date: 2017-09-07 19:43 -0700
http://bitbucket.org/pypy/pypy/changeset/46c6c05fde4b/

Log:	Explicitly use Py_ssize_t as the Signed type in pypy c-api.

	The signature for the c-api uses Py_ssize_t for the unsigned
	arguments. `long` is equivalent to `Py_ssize_t` (in fact
	`Py_ssize_t` is a `typedef` of `long`) in all cases in C, but when
	integrating with go, `C.long` and `C.Py_ssize_t` are considered two
	disparate incompatible types. By using `Py_ssize_t` in the function
	signature, pypy + cpython compatible code can be written.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1229,7 +1229,8 @@
 #  error "PyPy does not support 64-bit on Windows.  Use Win32"
 #endif
 ''',
-        '#define Signed   long           /* xxx temporary fix */',
+        '#include "cpyext_object.h"',
+        '#define Signed   Py_ssize_t     /* xxx temporary fix */',
         '#define Unsigned unsigned long  /* xxx temporary fix */',
         '',] + decls + [
         '',

From pypy.commits at gmail.com  Fri Sep  8 11:23:32 2017
From: pypy.commits at gmail.com (asottile)
Date: Fri, 08 Sep 2017 08:23:32 -0700 (PDT)
Subject: [pypy-commit] pypy py_ssize_t: Add `#pragma once` include guard to
 cpyext_object.h
Message-ID: <59b2b5f4.6588df0a.6a7a0.196b@mx.google.com>

Author: Anthony Sottile <asottile at umich.edu>
Branch: py_ssize_t
Changeset: r92357:44574602ff71
Date: 2017-09-08 07:35 -0700
http://bitbucket.org/pypy/pypy/changeset/44574602ff71/

Log:	Add `#pragma once` include guard to cpyext_object.h

	In order to get the definition of `Py_ssize_t`, other headers
	include this file. `#pragma once` was chosen as `pycparser` does not
	support parsing of preprocessor directives.

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,3 +1,4 @@
+#pragma once
 
 typedef long Py_ssize_t;
 

From pypy.commits at gmail.com  Fri Sep  8 11:23:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 08 Sep 2017 08:23:34 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge py_ssize_t
Message-ID: <59b2b5f6.81b5df0a.b22fa.ac40@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92358:ff810e9e706d
Date: 2017-09-08 17:21 +0200
http://bitbucket.org/pypy/pypy/changeset/ff810e9e706d/

Log:	hg merge py_ssize_t

	PR #568 (asottile)

	Trying to use another "#define Signed", together with "#pragma
	once", to make the cgo tool more happy. Might be reverted if
	"#pragma once" turns out not to work appropriately in all cases.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1229,7 +1229,8 @@
 #  error "PyPy does not support 64-bit on Windows.  Use Win32"
 #endif
 ''',
-        '#define Signed   long           /* xxx temporary fix */',
+        '#include "cpyext_object.h"',
+        '#define Signed   Py_ssize_t     /* xxx temporary fix */',
         '#define Unsigned unsigned long  /* xxx temporary fix */',
         '',] + decls + [
         '',
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,3 +1,4 @@
+#pragma once
 
 typedef long Py_ssize_t;
 

From pypy.commits at gmail.com  Fri Sep  8 11:23:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 08 Sep 2017 08:23:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <59b2b5f8.53bf1c0a.6ccc3.9945@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92359:a1608b11c5da
Date: 2017-09-08 17:22 +0200
http://bitbucket.org/pypy/pypy/changeset/a1608b11c5da/

Log:	merge heads

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -77,3 +77,7 @@
 .. branch: pypy_swappedbytes
 
 Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -29,10 +29,15 @@
         return default
     return space.is_true(w_src)
 
-def _get_int(space, w_src, default):
+def _get_int(space, w_src, default, attrname):
     if w_src is None:
         return default
-    return space.int_w(w_src)
+    try:
+        return space.int_w(w_src)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
+        raise
 
 def _get_str(space, w_src, default, attrname):
     if w_src is None:
@@ -100,7 +105,7 @@
     dialect.escapechar = _get_char(space, w_escapechar, '\0', 'escapechar')
     dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
     dialect.quotechar = _get_char(space, w_quotechar, '"', 'quotechar')
-    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL)
+    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
 
diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py
--- a/pypy/module/_csv/test/test_dialect.py
+++ b/pypy/module/_csv/test/test_dialect.py
@@ -65,7 +65,8 @@
             name = attempt[0]
             for value in attempt[1:]:
                 kwargs = {name: value}
-                raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                exc_info = raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                assert name in exc_info.value.args[0]
 
         exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4)
         assert exc_info.value.args[0] == '"lineterminator" must be a string'
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -129,6 +129,11 @@
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()
+
+for name in ('INT', 'LONG', 'LIST', 'TUPLE', 'UNICODE', 'DICT', 'BASE_EXC',
+             'TYPE', 'STRING'): # 'STRING' -> 'BYTES' in py3
+    constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name)
+
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
 globals().update(rffi_platform.configure(CConfig_constants))
@@ -749,6 +754,45 @@
 
     return check, check_exact
 
+def build_type_checkers_flags(type_name, cls=None, flagsubstr=None):
+    """
+    Builds two api functions: Py_XxxCheck() and Py_XxxCheckExact()
+    Does not export the functions, assumes they are macros in the *. files
+    check will try a fast path via pto flags
+    """
+    if cls is None:
+        attrname = "w_" + type_name.lower()
+        def get_w_type(space):
+            return getattr(space, attrname)
+    else:
+        def get_w_type(space):
+            return getattr(space, cls)
+    if flagsubstr is None:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % type_name.upper()
+    else:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % flagsubstr
+    check_name = "Py" + type_name + "_Check"
+    tp_flag = globals()[tp_flag_str]
+
+    @specialize.argtype(1)
+    def check(space, pto):
+        from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
+        "Implements the Py_Xxx_Check function"
+        if is_pyobj(pto):
+            return (pto.c_ob_type.c_tp_flags & tp_flag) == tp_flag
+        w_obj_type = space.type(pto)
+        w_type = get_w_type(space)
+        return (space.is_w(w_obj_type, w_type) or
+                space.issubtype_w(w_obj_type, w_type))
+
+    def check_exact(space, w_obj):
+        "Implements the Py_Xxx_CheckExact function"
+        w_obj_type = space.type(w_obj)
+        w_type = get_w_type(space)
+        return space.is_w(w_obj_type, w_type)
+
+    return check, check_exact
+
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
 
 
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
+    cpython_api, cpython_struct, bootstrap_function, build_type_checkers_flags,
     PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -58,7 +58,7 @@
                    dealloc=bytes_dealloc,
                    realize=bytes_realize)
 
-PyString_Check, PyString_CheckExact = build_type_checkers("String", "w_bytes")
+PyString_Check, PyString_CheckExact = build_type_checkers_flags("String", "w_bytes")
 
 def new_empty_str(space, length):
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -4,7 +4,7 @@
 from pypy.objspace.std.classdict import ClassDictStrategy
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
+    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
     bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
@@ -66,7 +66,7 @@
 def PyDict_New(space):
     return space.newdict()
 
-PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict")
+PyDict_Check, PyDict_CheckExact = build_type_checkers_flags("Dict")
 
 @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
diff --git a/pypy/module/cpyext/include/dictobject.h b/pypy/module/cpyext/include/dictobject.h
--- a/pypy/module/cpyext/include/dictobject.h
+++ b/pypy/module/cpyext/include/dictobject.h
@@ -12,6 +12,10 @@
     PyObject *_tmpkeys; /* a private place to put keys during PyDict_Next */
 } PyDictObject;
 
+#define PyDict_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_DICT_SUBCLASS)
+#define PyDict_CheckExact(op) ((op)->ob_type == &PyDict_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/intobject.h b/pypy/module/cpyext/include/intobject.h
--- a/pypy/module/cpyext/include/intobject.h
+++ b/pypy/module/cpyext/include/intobject.h
@@ -12,6 +12,10 @@
     long ob_ival;
 } PyIntObject;
 
+#define PyInt_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_INT_SUBCLASS)
+#define PyInt_CheckExact(op) ((op)->ob_type == &PyInt_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,4 @@
 /* empty */
+#define PyList_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS)
+#define PyList_CheckExact(op) ((op)->ob_type == &PyList_Type)
diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h
--- a/pypy/module/cpyext/include/longobject.h
+++ b/pypy/module/cpyext/include/longobject.h
@@ -14,6 +14,9 @@
 
 #define PyOS_strtoul strtoul
 #define PyOS_strtol strtoul
+#define PyLong_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS)
+#define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -236,6 +236,11 @@
 #define Py_TPFLAGS_DEFAULT Py_TPFLAGS_DEFAULT_EXTERNAL
 
 #define PyType_HasFeature(t,f)  (((t)->tp_flags & (f)) != 0)
+#define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
+
+#define PyType_Check(op) \
+    PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
+#define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type)
 
 /* objimpl.h ----------------------------------------------*/
 #define PyObject_New(type, typeobj) \
diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h
--- a/pypy/module/cpyext/include/pyerrors.h
+++ b/pypy/module/cpyext/include/pyerrors.h
@@ -9,7 +9,7 @@
 
 #define PyExceptionClass_Check(x)                                       \
     (PyClass_Check((x)) || (PyType_Check((x)) &&                        \
-      PyObject_IsSubclass((x), PyExc_BaseException)))
+      PyType_FastSubclass((PyTypeObject*)(x), Py_TPFLAGS_BASE_EXC_SUBCLASS)))
 
 PyAPI_FUNC(PyObject *) PyErr_NewException(const char *name, PyObject *base, PyObject *dict);
 PyAPI_FUNC(PyObject *) PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict);
diff --git a/pypy/module/cpyext/include/stringobject.h b/pypy/module/cpyext/include/stringobject.h
--- a/pypy/module/cpyext/include/stringobject.h
+++ b/pypy/module/cpyext/include/stringobject.h
@@ -61,6 +61,10 @@
 PyAPI_FUNC(PyObject *) PyString_FromFormatV(const char *format, va_list vargs);
 PyAPI_FUNC(PyObject *) PyString_FromFormat(const char *format, ...);
 
+#define PyString_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_STRING_SUBCLASS)
+#define PyString_CheckExact(op) ((op)->ob_type == &PyString_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -26,6 +26,9 @@
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+#define PyTuple_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_TUPLE_SUBCLASS)
+#define PyTuple_CheckExact(op) ((op)->ob_type == &PyTuple_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,10 @@
 
 #include "cpyext_unicodeobject.h"
 
+#define PyUnicode_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS)
+#define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/intobject.py b/pypy/module/cpyext/intobject.py
--- a/pypy/module/cpyext/intobject.py
+++ b/pypy/module/cpyext/intobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.interpreter.error import oefmt
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, build_type_checkers, bootstrap_function,
+    cpython_api, cpython_struct, build_type_checkers_flags, bootstrap_function,
     PyObject, PyObjectFields, CONST_STRING, CANNOT_FAIL, Py_ssize_t)
 from pypy.module.cpyext.pyobject import (
     make_typedescr, track_reference, from_ref)
@@ -40,7 +40,7 @@
     track_reference(space, obj, w_obj)
     return w_obj
 
-PyInt_Check, PyInt_CheckExact = build_type_checkers("Int")
+PyInt_Check, PyInt_CheckExact = build_type_checkers_flags("Int")
 
 @cpython_api([], lltype.Signed, error=CANNOT_FAIL)
 def PyInt_GetMax(space):
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -2,14 +2,14 @@
 from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
-                                    build_type_checkers)
+                                    build_type_checkers_flags)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
 
-PyList_Check, PyList_CheckExact = build_type_checkers("List")
+PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
 def PyList_New(space, len):
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,14 +1,13 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, build_type_checkers, Py_ssize_t,
+    cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
     CONST_STRING, ADDR, CANNOT_FAIL)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.intobject import PyInt_AsUnsignedLongMask
 from rpython.rlib.rbigint import rbigint
 
-
-PyLong_Check, PyLong_CheckExact = build_type_checkers("Long")
+PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long")
 
 @cpython_api([lltype.Signed], PyObject)
 def PyLong_FromLong(space, val):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    build_type_checkers, cpython_api, generic_cpy_call, CANNOT_FAIL,
+    cpython_api, generic_cpy_call, CANNOT_FAIL,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    cpython_api, bootstrap_function, cpython_struct, 
     slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1072,7 +1072,7 @@
 
     def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
-            ("fetchFooType", "METH_VARARGS",
+            ("fetchFooType", "METH_NOARGS",
              """
                 PyObject *o;
                 o = PyObject_New(PyObject, &Foo_Type);
@@ -1090,7 +1090,7 @@
                 Py_DECREF(e);
                 return o;
              """),
-            ("getCounter", "METH_VARARGS",
+            ("getCounter", "METH_NOARGS",
              """
                 return PyInt_FromLong(foo_counter);
              """)], prologue="""
@@ -1377,3 +1377,53 @@
         # this is equivalent to
         from collections import Hashable
         assert not isinstance(obj, Hashable)
+
+
+class AppTestFlags(AppTestCpythonExtensionBase):
+    def test_has_subclass_flag(self):
+        module = self.import_extension('foo', [
+           ("test_flags", "METH_VARARGS",
+            '''
+                long in_flag, my_flag;
+                PyObject * obj;
+                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                    return NULL;
+                if (!PyType_Check(obj))
+                {
+                    PyErr_SetString(PyExc_ValueError, "input must be type");
+                    return NULL;
+                }
+                my_flag = ((PyTypeObject*)obj)->tp_flags;
+                if ((my_flag & in_flag) != in_flag)
+                    return PyLong_FromLong(-1);
+                if (!PyType_CheckExact(obj)) {
+                    if ((my_flag & Py_TPFLAGS_TYPE_SUBCLASS) == Py_TPFLAGS_TYPE_SUBCLASS)
+                        return PyLong_FromLong(-2);
+                }
+                return PyLong_FromLong(0);
+            '''),])
+        # copied from object.h
+        Py_TPFLAGS_INT_SUBCLASS       = (1L<<23) # goes away on py3
+        Py_TPFLAGS_LONG_SUBCLASS      = (1L<<24)
+        Py_TPFLAGS_LIST_SUBCLASS      = (1L<<25)
+        Py_TPFLAGS_TUPLE_SUBCLASS     = (1L<<26)
+        Py_TPFLAGS_STRING_SUBCLASS    = (1L<<27) # rename to BYTES on py3
+        Py_TPFLAGS_UNICODE_SUBCLASS   = (1L<<28)
+        Py_TPFLAGS_DICT_SUBCLASS      = (1L<<29)
+        Py_TPFLAGS_BASE_EXC_SUBCLASS  = (1L<<30)
+        Py_TPFLAGS_TYPE_SUBCLASS      = (1L<<31)
+        for t,f in ((long, Py_TPFLAGS_LONG_SUBCLASS),
+                    (int, Py_TPFLAGS_INT_SUBCLASS),
+                    (list, Py_TPFLAGS_LIST_SUBCLASS),
+                    (tuple, Py_TPFLAGS_TUPLE_SUBCLASS),
+                    (str, Py_TPFLAGS_STRING_SUBCLASS),
+                    (unicode, Py_TPFLAGS_UNICODE_SUBCLASS),
+                    (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS),
+                    (type, Py_TPFLAGS_TYPE_SUBCLASS),
+                   ):
+            assert module.test_flags(t, f) == 0
+        class MyList(list):
+            pass
+        assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0
+
+         
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, build_type_checkers,
+    cpython_api, Py_ssize_t, build_type_checkers_flags,
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
@@ -42,7 +42,7 @@
                    dealloc=tuple_dealloc,
                    realize=tuple_realize)
 
-PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple")
+PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
 def tuple_check_ref(space, ref):
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -12,12 +12,17 @@
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
-    Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    Py_TPFLAGS_HAVE_GETCHARBUFFER, build_type_checkers,
-    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
-    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_CHECKTYPES,
-    Py_TPFLAGS_HAVE_INPLACEOPS, cts, parse_dir)
+    slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
+    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
+    PyTypeObjectPtr, Py_TPFLAGS_CHECKTYPES,
+    Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    Py_TPFLAGS_HAVE_GETCHARBUFFER, Py_TPFLAGS_HAVE_INPLACEOPS,
+    Py_TPFLAGS_HAVE_NEWBUFFER, Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
+    Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
+    Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
+    Py_TPFLAGS_TYPE_SUBCLASS,
+    Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
+    )
 from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -39,7 +44,7 @@
 
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
-PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
+PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
@@ -428,7 +433,7 @@
     dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space),
                                           from_ref(space, pyo), None)
 
-def inherit_special(space, pto, base_pto):
+def inherit_special(space, pto, w_obj, base_pto):
     # XXX missing: copy basicsize and flags in a magical way
     # (minimally, if tp_basicsize is zero or too low, we copy it from the base)
     if pto.c_tp_basicsize < base_pto.c_tp_basicsize:
@@ -438,6 +443,26 @@
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_CHECKTYPES
     pto.c_tp_flags |= base_pto.c_tp_flags & Py_TPFLAGS_HAVE_INPLACEOPS
 
+    #/* Setup fast subclass flags */
+    if space.issubtype_w(w_obj, space.w_Exception):
+        pto.c_tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_type):
+        pto.c_tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_int): # remove on py3
+        pto.c_tp_flags |= Py_TPFLAGS_INT_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_long):
+        pto.c_tp_flags |= Py_TPFLAGS_LONG_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_bytes): 
+        pto.c_tp_flags |= Py_TPFLAGS_STRING_SUBCLASS # STRING->BYTES on py3
+    elif space.issubtype_w(w_obj, space.w_unicode):
+        pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_tuple):
+        pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_list):
+        pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_dict):
+        pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS
+
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
         raise DescrMismatch()
@@ -939,7 +964,7 @@
     pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w))
     base = pto.c_tp_base
     if base:
-        inherit_special(space, pto, base)
+        inherit_special(space, pto, w_obj, base)
     for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)):
         if isinstance(w_base, W_TypeObject):
             inherit_slots(space, pto, w_base)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.unicodedata import unicodedb
 from pypy.module.cpyext.api import (
-    CANNOT_FAIL, Py_ssize_t, build_type_checkers, cpython_api,
+    CANNOT_FAIL, Py_ssize_t, build_type_checkers_flags, cpython_api,
     bootstrap_function, CONST_STRING,
     CONST_WSTRING, slot_function, cts, parse_dir)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
@@ -36,7 +36,7 @@
 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
                                  flavor='raw', zero=True)
 
-PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
+PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers_flags("Unicode")
 
 
 def new_empty_unicode(space, length):

From pypy.commits at gmail.com  Fri Sep  8 15:17:36 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 08 Sep 2017 12:17:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test for obscure difference with
 CPython
Message-ID: <59b2ecd0.41addf0a.a4baa.d742@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92360:44ba82e7527f
Date: 2017-09-08 20:17 +0100
http://bitbucket.org/pypy/pypy/changeset/44ba82e7527f/

Log:	Fix test for obscure difference with CPython (cf.
	test_builtin_reimport_mess() in pypy/module/imp/test/test_app.py)

diff --git a/lib-python/3/test/test_importlib/builtin/test_loader.py b/lib-python/3/test/test_importlib/builtin/test_loader.py
--- a/lib-python/3/test/test_importlib/builtin/test_loader.py
+++ b/lib-python/3/test/test_importlib/builtin/test_loader.py
@@ -1,6 +1,8 @@
 from .. import abc
 from .. import util
 
+from importlib.machinery import BuiltinImporter
+
 machinery = util.import_importlib('importlib.machinery')
 
 import sys
@@ -14,7 +16,7 @@
 
     def setUp(self):
         self.verification = {'__name__': 'errno', '__package__': '',
-                             '__loader__': self.machinery.BuiltinImporter}
+                             '__loader__': BuiltinImporter}  # PyPy change
 
     def verify(self, module):
         """Verify that the module matches against what it should have."""

From pypy.commits at gmail.com  Fri Sep  8 15:22:46 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 08 Sep 2017 12:22:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: use constants to make the code less
 confusing
Message-ID: <59b2ee06.8684df0a.9630c.e320@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92361:7de7062d9526
Date: 2017-09-08 17:01 +0200
http://bitbucket.org/pypy/pypy/changeset/7de7062d9526/

Log:	use constants to make the code less confusing

diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -10,6 +10,10 @@
 except ImportError:
     OrderedDict = dict # too bad
 
+SAVE_DEFAULT_REGS = 0
+SAVE_GCREF_REGS = 2
+SAVE_ALL_REGS = 1
+
 class TempVar(AbstractValue):
     def __init__(self):
         pass
@@ -586,7 +590,8 @@
                                                  force_store, save_all_regs)
 
     def spill_or_move_registers_before_call(self, save_sublist,
-                                            force_store=[], save_all_regs=0):
+                                            force_store=[],
+                                            save_all_regs=SAVE_DEFAULT_REGS):
         """Spill or move some registers before a call.
 
         By default, this means: for every register in 'save_sublist',
@@ -602,8 +607,9 @@
         valid, but only *if they are in self.save_around_call_regs,*
         not if they are callee-saved registers!
 
-        'save_all_regs' can be 0 (default set of registers), 1 (do that
-        for all registers), or 2 (default + gc ptrs).
+        'save_all_regs' can be SAVE_DEFAULT_REGS (default set of registers),
+        SAVE_ALL_REGS (do that for all registers), or SAVE_GCREF_REGS (default
+        + gc ptrs).
 
         Overview of what we do (the implementation does it differently,
         for the same result):
@@ -651,11 +657,11 @@
                 new_free_regs.append(reg)
                 continue
 
-            if save_all_regs == 1:
+            if save_all_regs == SAVE_ALL_REGS:
                 # we need to spill all registers in this mode
                 self._bc_spill(v, new_free_regs)
                 #
-            elif save_all_regs == 2 and v.type == REF:
+            elif save_all_regs == SAVE_GCREF_REGS and v.type == REF:
                 # we need to spill all GC ptrs in this mode
                 self._bc_spill(v, new_free_regs)
                 #
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -2,13 +2,13 @@
 """ Register allocation scheme.
 """
 
-import os, sys
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.llsupport.descr import CallDescr, unpack_arraydescr
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
      RegisterManager, TempVar, compute_vars_longevity, is_comparison_or_ovf_op,
-     valid_addressing_size, get_scale)
+     valid_addressing_size, get_scale, SAVE_DEFAULT_REGS, SAVE_GCREF_REGS,
+     SAVE_ALL_REGS)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
     IS_X86_64, DEFAULT_FRAME_BYTES)
@@ -23,12 +23,11 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr,
     ConstFloat, INT, REF, FLOAT, VECTOR, TargetToken, AbstractFailDescr)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.resume import AccumInfo
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.jit.backend.x86.regloc import AddressLoc
@@ -799,25 +798,29 @@
         # we need to save registers on the stack:
         #
         #  - at least the non-callee-saved registers
+        #    (gc_level == SAVE_DEFAULT_REGS)
         #
-        #  - if gc_level > 0, we save also the callee-saved registers that
-        #    contain GC pointers
+        #  - if gc_level == SAVE_GCREF_REGS we save also the callee-saved
+        #    registers that contain GC pointers
         #
-        #  - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER.  We
+        #  - gc_level == SAVE_ALL_REGS for CALL_MAY_FORCE or CALL_ASSEMBLER.  We
         #    have to save all regs anyway, in case we need to do
         #    cpu.force().  The issue is that grab_frame_values() would
         #    not be able to locate values in callee-saved registers.
         #
-        save_all_regs = gc_level == 2
+        if gc_level == SAVE_ALL_REGS:
+            save_all_regs = SAVE_ALL_REGS
+        else:
+            save_all_regs = SAVE_DEFAULT_REGS
         self.xrm.before_call(save_all_regs=save_all_regs)
-        if gc_level == 1:
+        if gc_level == SAVE_GCREF_REGS:
             gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
-            # we save all the registers for shadowstack and asmgcc for now
+            # we save all the GCREF registers for shadowstack and asmgcc for now
             # --- for asmgcc too: we can't say "register x is a gc ref"
             # without distinguishing call sites, which we don't do any
             # more for now.
             if gcrootmap: # and gcrootmap.is_shadow_stack:
-                save_all_regs = 2
+                save_all_regs = SAVE_GCREF_REGS
         self.rm.before_call(save_all_regs=save_all_regs)
         if op.type != 'v':
             if op.type == FLOAT:
@@ -841,11 +844,11 @@
         #
         effectinfo = calldescr.get_extra_info()
         if guard_not_forced:
-            gc_level = 2
+            gc_level = SAVE_ALL_REGS
         elif effectinfo is None or effectinfo.check_can_collect():
-            gc_level = 1
+            gc_level = SAVE_GCREF_REGS
         else:
-            gc_level = 0
+            gc_level = SAVE_DEFAULT_REGS
         #
         self._call(op, [imm(size), sign_loc] +
                        [self.loc(op.getarg(i)) for i in range(op.numargs())],
@@ -909,7 +912,7 @@
 
     def _consider_call_assembler(self, op):
         locs = self.locs_for_call_assembler(op)
-        self._call(op, locs, gc_level=2)
+        self._call(op, locs, gc_level=SAVE_ALL_REGS)
     consider_call_assembler_i = _consider_call_assembler
     consider_call_assembler_r = _consider_call_assembler
     consider_call_assembler_f = _consider_call_assembler

From pypy.commits at gmail.com  Sat Sep  9 09:01:54 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 09 Sep 2017 06:01:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix tests, hopefully
Message-ID: <59b3e642.08b51c0a.cdd38.0bb4@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92362:e2f4a15ea640
Date: 2017-09-09 15:01 +0200
http://bitbucket.org/pypy/pypy/changeset/e2f4a15ea640/

Log:	fix tests, hopefully

diff --git a/rpython/jit/codewriter/test/test_longlong.py b/rpython/jit/codewriter/test/test_longlong.py
--- a/rpython/jit/codewriter/test/test_longlong.py
+++ b/rpython/jit/codewriter/test/test_longlong.py
@@ -17,7 +17,7 @@
 class FakeBuiltinCallControl:
     def guess_call_kind(self, op):
         return 'builtin'
-    def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None):
+    def getcalldescr(self, op, oopspecindex=None, **kwargs):
         assert oopspecindex is not None    # in this test
         return 'calldescr-%d' % oopspecindex
     def calldescr_canraise(self, calldescr):

From pypy.commits at gmail.com  Sat Sep  9 11:00:17 2017
From: pypy.commits at gmail.com (fijal)
Date: Sat, 09 Sep 2017 08:00:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: bug from #2650
Message-ID: <59b40201.44b6df0a.705c9.7cb2@mx.google.com>

Author: fijal
Branch: 
Changeset: r92363:79a3ea5849ca
Date: 2017-09-09 16:59 +0200
http://bitbucket.org/pypy/pypy/changeset/79a3ea5849ca/

Log:	bug from #2650

diff --git a/rpython/jit/metainterp/test/test_loop.py b/rpython/jit/metainterp/test/test_loop.py
--- a/rpython/jit/metainterp/test/test_loop.py
+++ b/rpython/jit/metainterp/test/test_loop.py
@@ -1114,5 +1114,40 @@
         # one guard_false got removed
         self.check_resops(guard_false=4, guard_true=5)
 
+    def test_heapcache_bug(self):
+        class W_Object(object):
+            _attrs_ = []
+        class W_Nil(W_Object):
+            _attrs_ = []
+        class W_Cons(W_Object):
+            _attrs_ = ['first', 'rest']
+            _immutable_fields_ = ['first', 'rest']
+            def __init__(self, v1, v2):
+                self.first = v1
+                self.rest = v2
+
+        def reverse(xs):
+            result = W_Nil()
+            while isinstance(xs, W_Cons):
+                result = W_Cons(xs.first, result)
+                xs = xs.rest
+            return result
+
+        driver = JitDriver(reds=['repetitions', 'v'], greens=['pc'],
+                       get_printable_location=lambda pc: str(pc))
+        def entry_point():
+            repetitions = 0
+            while repetitions < 10:
+                pc = 0
+                v = W_Nil()
+                while pc < 10:
+                    driver.jit_merge_point(v=v, repetitions=repetitions, pc=pc)
+                    v = reverse(W_Cons(pc + 1, W_Cons(pc + 2, W_Cons(pc + 3, W_Cons(pc + 4, W_Nil())))))
+                    pc = pc + 1
+                repetitions += 1
+        
+        self.meta_interp(entry_point, [])
+
+
 class TestLLtype(LoopTest, LLJitMixin):
     pass

From pypy.commits at gmail.com  Sat Sep  9 16:41:22 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Sat, 09 Sep 2017 13:41:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: issue #2650 testing
Message-ID: <59b451f2.1986df0a.cb8d5.fdcc@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92364:0f364935cfcf
Date: 2017-09-09 22:40 +0200
http://bitbucket.org/pypy/pypy/changeset/0f364935cfcf/

Log:	issue #2650 testing

	Functions that write immutable fields don't need to invalidate the
	heap cache for such fields. However, they *do* need to force the
	lazy sets of such fields!

diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -445,13 +445,13 @@
             if effectinfo.check_readonly_descr_field(fielddescr):
                 cf.force_lazy_set(self, fielddescr)
             if effectinfo.check_write_descr_field(fielddescr):
+                cf.force_lazy_set(self, fielddescr, can_cache=False)
                 if fielddescr.is_always_pure():
                     continue
                 try:
                     del self.cached_dict_reads[fielddescr]
                 except KeyError:
                     pass
-                cf.force_lazy_set(self, fielddescr, can_cache=False)
         #
         for arraydescr, submap in self.cached_arrayitems.items():
             if effectinfo.check_readonly_descr_array(arraydescr):
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -3562,6 +3562,27 @@
         """
         self.optimize_loop(ops, expected, expected_preamble=expected_preamble)
 
+    def test_residual_call_still_forces_immutable_writes_though(self):
+        ops = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        expected_preamble = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected, expected_preamble=expected_preamble)
+
     def test_residual_call_invalidate_some_caches(self):
         ops = """
         [p1, p2]

From pypy.commits at gmail.com  Sun Sep 10 09:43:08 2017
From: pypy.commits at gmail.com (fijal)
Date: Sun, 10 Sep 2017 06:43:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: disable another pointless warning
Message-ID: <59b5416c.cc091c0a.6a43c.fb85@mx.google.com>

Author: fijal
Branch: 
Changeset: r92365:ee61202b4aa6
Date: 2017-09-10 15:42 +0200
http://bitbucket.org/pypy/pypy/changeset/ee61202b4aa6/

Log:	disable another pointless warning

diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -151,8 +151,9 @@
                 # Also, ERROR confuses lib-python/conftest.py.
             raise CompilationError(stdout, stderr)
         else:
-            for line in stderr.splitlines():
-                log.WARNING(line)
+            if self.log_errors:
+                for line in stderr.splitlines():
+                    log.WARNING(line)
 
     def _make_o_file(self, cfile, ext):
         """Create an object file name under the udir for a .c file"""

From pypy.commits at gmail.com  Mon Sep 11 03:04:58 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 11 Sep 2017 00:04:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: really fix tests
Message-ID: <59b6359a.928e1c0a.883f3.9499@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92366:361e12f80c69
Date: 2017-09-11 09:04 +0200
http://bitbucket.org/pypy/pypy/changeset/361e12f80c69/

Log:	really fix tests

diff --git a/rpython/jit/codewriter/test/test_longlong.py b/rpython/jit/codewriter/test/test_longlong.py
--- a/rpython/jit/codewriter/test/test_longlong.py
+++ b/rpython/jit/codewriter/test/test_longlong.py
@@ -17,7 +17,7 @@
 class FakeBuiltinCallControl:
     def guess_call_kind(self, op):
         return 'builtin'
-    def getcalldescr(self, op, oopspecindex=None, **kwargs):
+    def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None, **kwargs):
         assert oopspecindex is not None    # in this test
         return 'calldescr-%d' % oopspecindex
     def calldescr_canraise(self, calldescr):

From pypy.commits at gmail.com  Mon Sep 11 11:45:25 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 08:45:25 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tp_dictoffset: set tp_dictoffset,
 used only as hint for cython
Message-ID: <59b6af95.53bf1c0a.6ccc3.bb60@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-tp_dictoffset
Changeset: r92368:54a0a8a5a9d9
Date: 2017-09-11 18:09 +0300
http://bitbucket.org/pypy/pypy/changeset/54a0a8a5a9d9/

Log:	set tp_dictoffset, used only as hint for cython


From pypy.commits at gmail.com  Mon Sep 11 11:45:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 08:45:27 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tp_dictoffset: test,
 fix setting tp_dictoffset as indicator for mutable attributes
 (cython)
Message-ID: <59b6af97.50131c0a.7e024.5b8d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-tp_dictoffset
Changeset: r92369:56afd2a1b1e9
Date: 2017-09-09 23:58 +0300
http://bitbucket.org/pypy/pypy/changeset/56afd2a1b1e9/

Log:	test, fix setting tp_dictoffset as indicator for mutable attributes
	(cython)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1206,6 +1206,8 @@
     py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
                                  make_ref(space, w_type))
     typedescr.attach(space, py_obj, w_obj)
+    # prevent PyObject_SetAttr on built-ins
+    rffi.cast(PyTypeObjectPtr, py_obj).c_tp_dictoffset = 0
     attached_objs.append(i)
 
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -6,7 +6,7 @@
     Py_GE, CONST_STRING, FILEP, fwrite)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, from_ref, Py_IncRef, Py_DecRef,
-    get_typedescr)
+    get_typedescr, as_pyobj)
 from pypy.module.cpyext.typeobject import PyTypeObjectPtr
 from pypy.module.cpyext.pyerrors import PyErr_NoMemory, PyErr_BadInternalCall
 from pypy.objspace.std.typeobject import W_TypeObject
@@ -122,12 +122,20 @@
 
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyObject_SetAttr(space, w_obj, w_name, w_value):
+    pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+        raise oefmt(space.w_AttributeError,
+             "object has no attribute %s", space.text_w(w_name))
     operation.setattr(space, w_obj, w_name, w_value)
     return 0
 
 @cpython_api([PyObject, CONST_STRING, PyObject], rffi.INT_real, error=-1)
 def PyObject_SetAttrString(space, w_obj, name_ptr, w_value):
     w_name = space.newtext(rffi.charp2str(name_ptr))
+    pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+        raise oefmt(space.w_AttributeError,
+             "object has no attribute %s", space.text_w(w_name))
     operation.setattr(space, w_obj, w_name, w_value)
     return 0
 
@@ -135,6 +143,10 @@
 def PyObject_DelAttr(space, w_obj, w_name):
     """Delete attribute named attr_name, for object o. Returns -1 on failure.
     This is the equivalent of the Python statement del o.attr_name."""
+    pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+        raise oefmt(space.w_AttributeError,
+             "object has no attribute %s", space.text_w(w_name))
     space.delattr(w_obj, w_name)
     return 0
 
@@ -143,6 +155,10 @@
     """Delete attribute named attr_name, for object o. Returns -1 on failure.
     This is the equivalent of the Python statement del o.attr_name."""
     w_name = space.newtext(rffi.charp2str(name_ptr))
+    pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+        raise oefmt(space.w_AttributeError,
+             "object has no attribute %s", space.text_w(w_name))
     space.delattr(w_obj, w_name)
     return 0
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -276,7 +276,7 @@
                  PyObject *method;
                  if (!args->ob_type->tp_dict)
                  {
-                     PyErr_SetNone(PyExc_ValueError);
+                     PyErr_SetString(PyExc_ValueError, "no tp_dict");
                      return NULL;
                  }
                  method = PyDict_GetItemString(
@@ -291,25 +291,54 @@
                 Py_INCREF(value);
                 return value;
              '''),
+            ('set_attr', "METH_VARARGS",
+             '''
+                PyObject * v, *name, *w;
+                int ret;
+                if (!PyArg_ParseTuple(args, "OOO", &v, &name, &w))
+                    return NULL;
+                ret = PyObject_SetAttr(v, name, w);
+                if (ret != 0)
+                    return NULL;
+                Py_RETURN_NONE;
+             '''),   
+            ('get_tp_dictoffset', "METH_O",
+             '''
+                return PyLong_FromLong(args->ob_type->tp_dictoffset);
+             '''),
             ])
         obj = foo.new()
+        assert module.get_tp_dictoffset(obj) == 0
         assert module.read_tp_dict(obj) == foo.fooType.copy
         d = module.get_type_dict(obj)
         assert type(d) is dict
         d["_some_attribute"] = 1
         assert type(obj)._some_attribute == 1
+        assert module.get_tp_dictoffset(obj) == 0
+        del d["_some_attribute"]
+        assert module.get_tp_dictoffset(obj) == 0
+
+        def method(self):
+            return 42
+
+        exc = raises(AttributeError, module.set_attr, obj, 'meth', method)
+        assert 'object has no attribute' in str(exc.value)
+
+        class A(object):
+            pass
+        obj = A()
+        assert module.get_tp_dictoffset(obj) > 0
+        module.set_attr(obj, 'meth', method)
+        assert obj.meth(obj) == 42
+        d = module.get_type_dict(obj)
+        assert type(d) is dict
+        d["_some_attribute"] = 1
+        assert type(obj)._some_attribute == 1
         del d["_some_attribute"]
 
-        class A(object):
-            pass
-        obj = A()
-        d = module.get_type_dict(obj)
-        assert type(d) is dict
-        d["_some_attribute"] = 1
-        assert type(obj)._some_attribute == 1
-        del d["_some_attribute"]
-
-        d = module.get_type_dict(1)
+        a = 1
+        d = module.get_type_dict(a)
+        assert module.get_tp_dictoffset(a) == 0
         assert type(d) is dict
         try:
             d["_some_attribute"] = 1
@@ -318,6 +347,7 @@
         else:
             assert int._some_attribute == 1
             del d["_some_attribute"]
+        assert module.get_tp_dictoffset(a) == 0
 
     def test_custom_allocation(self):
         foo = self.import_module("foo")
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -815,6 +815,7 @@
         if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
                 pto.c_tp_new = pto.c_tp_base.c_tp_new
         Py_DecRef(space, base_object_pyo)
+    pto.c_tp_dictoffset = rffi.offsetof(lltype.typeOf(pto).TO, 'c_tp_dictoffset')
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
 

From pypy.commits at gmail.com  Mon Sep 11 11:45:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 08:45:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: test,
 implement PyUnicode_FromFormat in C
Message-ID: <59b6af90.8684df0a.9630c.c677@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92367:e3a231de06fb
Date: 2017-09-09 23:55 +0300
http://bitbucket.org/pypy/pypy/changeset/e3a231de06fb/

Log:	test, implement PyUnicode_FromFormat in C

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -540,6 +540,7 @@
     'PyArg_ParseTuple', 'PyArg_UnpackTuple', 'PyArg_ParseTupleAndKeywords',
     'PyArg_VaParse', 'PyArg_VaParseTupleAndKeywords', '_PyArg_NoKeywords',
     'PyString_FromFormat', 'PyString_FromFormatV',
+    'PyUnicode_FromFormat', 'PyUnicode_FromFormatV',
     'PyModule_AddObject', 'PyModule_AddIntConstant', 'PyModule_AddStringConstant',
     'Py_BuildValue', 'Py_VaBuildValue', 'PyTuple_Pack',
     '_PyArg_Parse_SizeT', '_PyArg_ParseTuple_SizeT',
@@ -1340,6 +1341,7 @@
                          source_dir / "getargs.c",
                          source_dir / "abstract.c",
                          source_dir / "stringobject.c",
+                         source_dir / "unicodeobject.c",
                          source_dir / "mysnprintf.c",
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,9 @@
 
 #include "cpyext_unicodeobject.h"
 
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormatV(const char *format, va_list vargs);
+PyAPI_FUNC(PyObject *) PyUnicode_FromFormat(const char *format, ...);
+
 #define PyUnicode_Check(op) \
 		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS)
 #define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type)
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1500,151 +1500,6 @@
     """
     raise NotImplementedError
 
- at cpython_api([rffi.CCHARP], PyObject)
-def PyUnicode_FromFormat(space, format):
-    """Take a C printf()-style format string and a variable number of
-    arguments, calculate the size of the resulting Python unicode string and return
-    a string with the values formatted into it.  The variable arguments must be C
-    types and must correspond exactly to the format characters in the format
-    string.  The following format characters are allowed:
-
-    Format Characters
-
-    Type
-
-    Comment
-
-    %%
-
-    n/a
-
-    The literal % character.
-
-    %c
-
-    int
-
-    A single character,
-    represented as an C int.
-
-    %d
-
-    int
-
-    Exactly equivalent to
-    printf("%d").
-
-    %u
-
-    unsigned int
-
-    Exactly equivalent to
-    printf("%u").
-
-    %ld
-
-    long
-
-    Exactly equivalent to
-    printf("%ld").
-
-    %lu
-
-    unsigned long
-
-    Exactly equivalent to
-    printf("%lu").
-
-    %zd
-
-    Py_ssize_t
-
-    Exactly equivalent to
-    printf("%zd").
-
-    %zu
-
-    size_t
-
-    Exactly equivalent to
-    printf("%zu").
-
-    %i
-
-    int
-
-    Exactly equivalent to
-    printf("%i").
-
-    %x
-
-    int
-
-    Exactly equivalent to
-    printf("%x").
-
-    %s
-
-    char*
-
-    A null-terminated C character
-    array.
-
-    %p
-
-    void*
-
-    The hex representation of a C
-    pointer. Mostly equivalent to
-    printf("%p") except that
-    it is guaranteed to start with
-    the literal 0x regardless
-    of what the platform's
-    printf yields.
-
-    %U
-
-    PyObject*
-
-    A unicode object.
-
-    %V
-
-    PyObject*, char *
-
-    A unicode object (which may be
-    NULL) and a null-terminated
-    C character array as a second
-    parameter (which will be used,
-    if the first parameter is
-    NULL).
-
-    %S
-
-    PyObject*
-
-    The result of calling
-    PyObject_Unicode().
-
-    %R
-
-    PyObject*
-
-    The result of calling
-    PyObject_Repr().
-
-    An unrecognized format character causes all the rest of the format string to be
-    copied as-is to the result string, and any extra arguments discarded.
-    """
-    raise NotImplementedError
-
- at cpython_api([rffi.CCHARP, va_list], PyObject)
-def PyUnicode_FromFormatV(space, format, vargs):
-    """Identical to PyUnicode_FromFormat() except that it takes exactly two
-    arguments.
-    """
-    raise NotImplementedError
-
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyUnicode_Encode(space, s, size, encoding, errors):
     """Encode the Py_UNICODE buffer of the given size and return a Python
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -132,6 +132,20 @@
              """)])
         assert module.test_macro_invocations() == u''
 
+    def test_format(self):
+        module = self.import_extension('foo', [
+            ("test_unicode_format", "METH_VARARGS",
+             '''
+                 return PyUnicode_FromFormat("bla %d ble %s\\n",
+                        PyInt_AsLong(PyTuple_GetItem(args, 0)),
+                        PyString_AsString(PyTuple_GetItem(args, 1)));
+             '''
+             )
+            ])
+        res = module.test_unicode_format(1, "xyz")
+        assert res == u"bla 1 ble xyz\n"
+
+
 class TestUnicode(BaseApiTest):
     def test_unicodeobject(self, space):
         assert PyUnicode_GET_SIZE(space, space.wrap(u'sp&#65533;m')) == 4

From pypy.commits at gmail.com  Mon Sep 11 11:45:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 08:45:29 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-tp_dictoffset: test,
 fixes for when tp_dictoffset is non-zero, when PyObject_*Attr* can run
Message-ID: <59b6af99.cc091c0a.67e93.6135@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cpyext-tp_dictoffset
Changeset: r92370:e2b3bea05c2b
Date: 2017-09-11 18:06 +0300
http://bitbucket.org/pypy/pypy/changeset/e2b3bea05c2b/

Log:	test, fixes for when tp_dictoffset is non-zero, when PyObject_*Attr*
	can run

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1183,6 +1183,7 @@
 def attach_recursively(space, static_pyobjs, static_objs_w, attached_objs, i):
     # Start at i but make sure all the base classes are already attached
     from pypy.module.cpyext.pyobject import get_typedescr, make_ref
+    from pypy.module.cpyext.typeobject import type_attach
     if i in attached_objs:
         return
     py_obj = static_pyobjs[i]
@@ -1205,9 +1206,18 @@
     typedescr = get_typedescr(w_type.layout.typedef)
     py_obj.c_ob_type = rffi.cast(PyTypeObjectPtr,
                                  make_ref(space, w_type))
-    typedescr.attach(space, py_obj, w_obj)
-    # prevent PyObject_SetAttr on built-ins
-    rffi.cast(PyTypeObjectPtr, py_obj).c_tp_dictoffset = 0
+    if isinstance(w_obj, W_TypeObject):
+        # type and module objects need to have tp_dictoffset != 0
+        # so that, like user-defined python classes, they can have
+        # attributes set via PyObject_SetAttr()
+        set_tp_dictoffset = False
+        if w_obj is space.w_type:
+            set_tp_dictoffset = True
+        if space.is_w(w_obj, space.gettypeobject(Module.typedef)):
+            set_tp_dictoffset = True
+        type_attach(space, py_obj, w_obj, set_tp_dictoffset=set_tp_dictoffset)
+    else:
+        typedescr.attach(space, py_obj, w_obj) 
     attached_objs.append(i)
 
 
diff --git a/pypy/module/cpyext/object.py b/pypy/module/cpyext/object.py
--- a/pypy/module/cpyext/object.py
+++ b/pypy/module/cpyext/object.py
@@ -123,7 +123,9 @@
 @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
 def PyObject_SetAttr(space, w_obj, w_name, w_value):
     pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
-    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0 and
+            not pyobj.c_ob_type.c_tp_setattro and
+            not pyobj.c_ob_type.c_tp_setattr):
         raise oefmt(space.w_AttributeError,
              "object has no attribute %s", space.text_w(w_name))
     operation.setattr(space, w_obj, w_name, w_value)
@@ -133,7 +135,9 @@
 def PyObject_SetAttrString(space, w_obj, name_ptr, w_value):
     w_name = space.newtext(rffi.charp2str(name_ptr))
     pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
-    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0 and
+            not pyobj.c_ob_type.c_tp_setattro and
+            not pyobj.c_ob_type.c_tp_setattr):
         raise oefmt(space.w_AttributeError,
              "object has no attribute %s", space.text_w(w_name))
     operation.setattr(space, w_obj, w_name, w_value)
@@ -144,7 +148,9 @@
     """Delete attribute named attr_name, for object o. Returns -1 on failure.
     This is the equivalent of the Python statement del o.attr_name."""
     pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
-    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0 and
+            not pyobj.c_ob_type.c_tp_setattro and
+            not pyobj.c_ob_type.c_tp_setattr):
         raise oefmt(space.w_AttributeError,
              "object has no attribute %s", space.text_w(w_name))
     space.delattr(w_obj, w_name)
@@ -156,7 +162,9 @@
     This is the equivalent of the Python statement del o.attr_name."""
     w_name = space.newtext(rffi.charp2str(name_ptr))
     pyobj = as_pyobj(space, w_obj) # assumes w_obj is kept alive for the call
-    if (pyobj.c_ob_type.c_tp_dictoffset == 0):
+    if (pyobj.c_ob_type.c_tp_dictoffset == 0 and
+            not pyobj.c_ob_type.c_tp_setattro and
+            not pyobj.c_ob_type.c_tp_setattr):
         raise oefmt(space.w_AttributeError,
              "object has no attribute %s", space.text_w(w_name))
     space.delattr(w_obj, w_name)
diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -584,12 +584,17 @@
             Py_INCREF(t);
             return t;
         }
+        static PyObject * foo_get_dictoffset(PyObject *self, PyObject * args) {
+            return PyLong_FromLong(args->ob_type->tp_dictoffset);
+        }
         static PyMethodDef methods[] = {
             { "test", foo_test, METH_VARARGS },
+            { "get_dictoffset",   foo_get_dictoffset, METH_O },
             { NULL }
         };
         """
         module = self.import_module(name='foo', init=init, body=body)
+        assert module.get_dictoffset(module) > 0
         assert module.test(True, True) == True
 
     def test_exception(self):
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -388,6 +388,10 @@
                         return NULL;
                     }
                     return args->ob_type->tp_iternext(args);
+                 '''),
+                ('get_dictoffset', "METH_O",
+                 '''
+                    return PyLong_FromLong(args->ob_type->tp_dictoffset);
                  '''),])
             def __init__(self, N):
                 self.N = N
@@ -415,3 +419,4 @@
             except StopIteration:
                 pass
             assert out == [0, 1, 2, 3, 4]
+            assert module.get_dictoffset(c) > 0
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -569,18 +569,18 @@
     # w_obj is an instance of w_A or one of its subclasses. So climb up the
     # inheritance chain until base.c_tp_dealloc is exactly this_func, and then
     # continue on up until they differ.
-    #print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
+    # print 'subtype_dealloc, start from', rffi.charp2str(base.c_tp_name)
     while base.c_tp_dealloc != this_func_ptr:
         base = base.c_tp_base
         assert base
-        #print '                 ne move to', rffi.charp2str(base.c_tp_name)
+        # print '                 ne move to', rffi.charp2str(base.c_tp_name)
         w_obj = from_ref(space, rffi.cast(PyObject, base))
     while base.c_tp_dealloc == this_func_ptr:
         base = base.c_tp_base
         assert base
-        #print '                 eq move to', rffi.charp2str(base.c_tp_name)
+        # print '                 eq move to', rffi.charp2str(base.c_tp_name)
         w_obj = from_ref(space, rffi.cast(PyObject, base))
-    #print '                   end with', rffi.charp2str(base.c_tp_name)
+    # print '                   end with', rffi.charp2str(base.c_tp_name)
     dealloc = base.c_tp_dealloc
     # XXX call tp_del if necessary
     generic_cpy_call(space, dealloc, obj)
@@ -740,7 +740,7 @@
 
     return rffi.cast(PyObject, heaptype)
 
-def type_attach(space, py_obj, w_type, w_userdata=None):
+def type_attach(space, py_obj, w_type, w_userdata=None, set_tp_dictoffset=True):
     """
     Fills a newly allocated PyTypeObject from an existing type.
     """
@@ -815,7 +815,8 @@
         if pto.c_tp_base != base_object_pto or flags & Py_TPFLAGS_HEAPTYPE:
                 pto.c_tp_new = pto.c_tp_base.c_tp_new
         Py_DecRef(space, base_object_pyo)
-    pto.c_tp_dictoffset = rffi.offsetof(lltype.typeOf(pto).TO, 'c_tp_dictoffset')
+    if set_tp_dictoffset:
+        pto.c_tp_dictoffset = rffi.offsetof(lltype.typeOf(pto).TO, 'c_tp_dictoffset')
     pto.c_tp_flags |= Py_TPFLAGS_READY
     return pto
 

From pypy.commits at gmail.com  Mon Sep 11 11:45:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 08:45:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: copy-paste typo
Message-ID: <59b6af9b.93aa1c0a.e4a68.ca26@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92371:58204c4f3efd
Date: 2017-09-11 18:40 +0300
http://bitbucket.org/pypy/pypy/changeset/58204c4f3efd/

Log:	copy-paste typo

diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -36,7 +36,7 @@
         return space.int_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
+            raise oefmt(space.w_TypeError, '"%s" must be an int', attrname)
         raise
 
 def _get_str(space, w_src, default, attrname):

From pypy.commits at gmail.com  Mon Sep 11 12:54:20 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 11 Sep 2017 09:54:20 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default (fast PyUnicode_Check
 disabled for now)
Message-ID: <59b6bfbc.03c6df0a.e25a6.8b34@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92372:1f0dbee16f1f
Date: 2017-09-11 17:53 +0100
http://bitbucket.org/pypy/pypy/changeset/1f0dbee16f1f/

Log:	hg merge default (fast PyUnicode_Check disabled for now)

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -73,3 +73,11 @@
 
 Add support for leakfinder in cpyext tests (disabled for now, due to too many
 failures).
+
+.. branch: pypy_swappedbytes
+
+Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1537,12 +1537,15 @@
     def text_or_none_w(self, w_obj):
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    @specialize.argtype(1)
     def bytes_w(self, w_obj):
         """ Takes an application level :py:class:`bytes`
             (on PyPy2 this equals `str`) and returns a rpython byte string.
         """
+        assert w_obj is not None
         return w_obj.bytes_w(self)
 
+    @specialize.argtype(1)
     def text_w(self, w_obj):
         """ PyPy2 takes either a :py:class:`str` and returns a
             rpython byte string, or it takes an :py:class:`unicode`
@@ -1552,6 +1555,7 @@
             On PyPy3 it takes a :py:class:`str` and it will return
             an utf-8 encoded rpython string.
         """
+        assert w_obj is not None
         return w_obj.text_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1601,6 +1605,7 @@
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
         return chr(value)
 
+    @specialize.argtype(1)
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1613,29 +1618,40 @@
         If allow_conversion=False, w_obj needs to be an app-level int or a
         subclass.
         """
+        assert w_obj is not None
         return w_obj.int_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def int(self, w_obj):
+        assert w_obj is not None
         return w_obj.int(self)
 
+    @specialize.argtype(1)
     def uint_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.uint_w(self)
 
+    @specialize.argtype(1)
     def bigint_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return a rlib.rbigint object and call __long__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.bigint_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def float_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return an interp-level float and call __float__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.float_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def unicode_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.unicode_w(self)
 
     def unicode0_w(self, w_obj):
@@ -1690,7 +1706,9 @@
         # this, but the general is_true(),  accepting any object.
         return bool(self.int_w(w_obj))
 
+    @specialize.argtype(1)
     def ord(self, w_obj):
+        assert w_obj is not None
         return w_obj.ord(self)
 
     # This is all interface for gateway.py.
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -29,10 +29,15 @@
         return default
     return space.is_true(w_src)
 
-def _get_int(space, w_src, default):
+def _get_int(space, w_src, default, attrname):
     if w_src is None:
         return default
-    return space.int_w(w_src)
+    try:
+        return space.int_w(w_src)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
+        raise
 
 def _get_str(space, w_src, default, attrname):
     if w_src is None:
@@ -100,7 +105,7 @@
     dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar')
     dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator')
     dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar')
-    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL)
+    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
 
diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py
--- a/pypy/module/_csv/test/test_dialect.py
+++ b/pypy/module/_csv/test/test_dialect.py
@@ -65,7 +65,8 @@
             name = attempt[0]
             for value in attempt[1:]:
                 kwargs = {name: value}
-                raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                exc_info = raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                assert name in exc_info.value.args[0]
 
         exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4)
         assert exc_info.value.args[0] == '"lineterminator" must be a string'
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
@@ -129,6 +130,11 @@
 Py_CLEANUP_SUPPORTED
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE PyBUF_SIMPLE PyBUF_WRITE
 """.split()
+
+for name in ('LONG', 'LIST', 'TUPLE', 'UNICODE', 'DICT', 'BASE_EXC',
+             'TYPE', 'BYTES'):
+    constant_names.append('Py_TPFLAGS_%s_SUBCLASS' % name)
+
 for name in constant_names:
     setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name))
 globals().update(rffi_platform.configure(CConfig_constants))
@@ -765,6 +771,45 @@
 
     return check, check_exact
 
+def build_type_checkers_flags(type_name, cls=None, flagsubstr=None):
+    """
+    Builds two api functions: Py_XxxCheck() and Py_XxxCheckExact()
+    Does not export the functions, assumes they are macros in the *. files
+    check will try a fast path via pto flags
+    """
+    if cls is None:
+        attrname = "w_" + type_name.lower()
+        def get_w_type(space):
+            return getattr(space, attrname)
+    else:
+        def get_w_type(space):
+            return getattr(space, cls)
+    if flagsubstr is None:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % type_name.upper()
+    else:
+       tp_flag_str = 'Py_TPFLAGS_%s_SUBCLASS' % flagsubstr
+    check_name = "Py" + type_name + "_Check"
+    tp_flag = globals()[tp_flag_str]
+
+    @specialize.argtype(1)
+    def check(space, pto):
+        from pypy.module.cpyext.pyobject import is_pyobj, as_pyobj
+        "Implements the Py_Xxx_Check function"
+        if is_pyobj(pto):
+            return (pto.c_ob_type.c_tp_flags & tp_flag) == tp_flag
+        w_obj_type = space.type(pto)
+        w_type = get_w_type(space)
+        return (space.is_w(w_obj_type, w_type) or
+                space.issubtype_w(w_obj_type, w_type))
+
+    def check_exact(space, w_obj):
+        "Implements the Py_Xxx_CheckExact function"
+        w_obj_type = space.type(w_obj)
+        w_type = get_w_type(space)
+        return space.is_w(w_obj_type, w_type)
+
+    return check, check_exact
+
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
 
 
@@ -957,6 +1002,11 @@
                     message = str(e)
                 state.set_exception(OperationError(space.w_SystemError,
                                                    space.newtext(message)))
+            except rstackovf.StackOverflow as e:
+                rstackovf.check_stack_overflow()
+                failed = True
+                state.set_exception(OperationError(space.w_RuntimeError,
+                         space.newtext("maximum recursion depth exceeded")))
             else:
                 failed = False
 
@@ -1240,7 +1290,8 @@
 #  error "PyPy does not support 64-bit on Windows.  Use Win32"
 #endif
 ''',
-        '#define Signed   long           /* xxx temporary fix */',
+        '#include "cpyext_object.h"',
+        '#define Signed   Py_ssize_t     /* xxx temporary fix */',
         '#define Unsigned unsigned long  /* xxx temporary fix */',
         '',] + decls + [
         '',
diff --git a/pypy/module/cpyext/bytesobject.py b/pypy/module/cpyext/bytesobject.py
--- a/pypy/module/cpyext/bytesobject.py
+++ b/pypy/module/cpyext/bytesobject.py
@@ -1,7 +1,7 @@
 from pypy.interpreter.error import oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, bootstrap_function, build_type_checkers,
+    cpython_api, cpython_struct, bootstrap_function, build_type_checkers_flags,
     PyVarObjectFields, Py_ssize_t, CONST_STRING, CANNOT_FAIL, slot_function)
 from pypy.module.cpyext.pyerrors import PyErr_BadArgument
 from pypy.module.cpyext.pyobject import (
@@ -58,7 +58,7 @@
                    dealloc=bytes_dealloc,
                    realize=bytes_realize)
 
-PyBytes_Check, PyBytes_CheckExact = build_type_checkers("Bytes", "w_bytes")
+PyBytes_Check, PyBytes_CheckExact = build_type_checkers_flags("Bytes", "w_bytes")
 
 def new_empty_str(space, length):
     """
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -4,7 +4,7 @@
 from pypy.objspace.std.classdict import ClassDictStrategy
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, build_type_checkers, Py_ssize_t,
+    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
     bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
@@ -66,7 +66,7 @@
 def PyDict_New(space):
     return space.newdict()
 
-PyDict_Check, PyDict_CheckExact = build_type_checkers("Dict")
+PyDict_Check, PyDict_CheckExact = build_type_checkers_flags("Dict")
 
 @cpython_api([PyObject, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
diff --git a/pypy/module/cpyext/include/bytesobject.h b/pypy/module/cpyext/include/bytesobject.h
--- a/pypy/module/cpyext/include/bytesobject.h
+++ b/pypy/module/cpyext/include/bytesobject.h
@@ -55,6 +55,9 @@
 #define SSTATE_INTERNED_IMMORTAL 2
 #define PyString_CHECK_INTERNED(op) (((PyStringObject *)(op))->ob_sstate)
 
+#define PyBytes_Check(op) \
+                 PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_BYTES_SUBCLASS)
+#define PyBytes_CheckExact(op) (Py_TYPE(op) == &PyBytes_Type)
 
 PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list);
 PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...);
diff --git a/pypy/module/cpyext/include/dictobject.h b/pypy/module/cpyext/include/dictobject.h
--- a/pypy/module/cpyext/include/dictobject.h
+++ b/pypy/module/cpyext/include/dictobject.h
@@ -12,6 +12,10 @@
     PyObject *_tmpkeys; /* a private place to put keys during PyDict_Next */
 } PyDictObject;
 
+#define PyDict_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_DICT_SUBCLASS)
+#define PyDict_CheckExact(op) ((op)->ob_type == &PyDict_Type)
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/pypy/module/cpyext/include/listobject.h b/pypy/module/cpyext/include/listobject.h
--- a/pypy/module/cpyext/include/listobject.h
+++ b/pypy/module/cpyext/include/listobject.h
@@ -1,1 +1,4 @@
 /* empty */
+#define PyList_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LIST_SUBCLASS)
+#define PyList_CheckExact(op) ((op)->ob_type == &PyList_Type)
diff --git a/pypy/module/cpyext/include/longobject.h b/pypy/module/cpyext/include/longobject.h
--- a/pypy/module/cpyext/include/longobject.h
+++ b/pypy/module/cpyext/include/longobject.h
@@ -14,6 +14,9 @@
 
 #define PyOS_strtoul strtoul
 #define PyOS_strtol strtoul
+#define PyLong_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_LONG_SUBCLASS)
+#define PyLong_CheckExact(op) ((op)->ob_type == &PyLong_Type)
 
 #define PyLong_AS_LONG(op) PyLong_AsLong(op)
 
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -227,6 +227,10 @@
 #endif
 #define PyType_FastSubclass(t,f)  PyType_HasFeature(t,f)
 
+#define PyType_Check(op) \
+    PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_TYPE_SUBCLASS)
+#define PyType_CheckExact(op) (Py_TYPE(op) == &PyType_Type)
+
 /* objimpl.h ----------------------------------------------*/
 #define PyObject_New(type, typeobj) \
 		( (type *) _PyObject_New(typeobj) )
diff --git a/pypy/module/cpyext/include/pyerrors.h b/pypy/module/cpyext/include/pyerrors.h
--- a/pypy/module/cpyext/include/pyerrors.h
+++ b/pypy/module/cpyext/include/pyerrors.h
@@ -9,7 +9,7 @@
 
 #define PyExceptionClass_Check(x)                                       \
     ((PyType_Check((x)) &&                                              \
-      PyObject_IsSubclass((x), PyExc_BaseException)))
+      PyType_FastSubclass((PyTypeObject*)(x), Py_TPFLAGS_BASE_EXC_SUBCLASS)))
 
 #define PyExceptionInstance_Check(x)                                    \
     (PyObject_IsSubclass((PyObject *)Py_TYPE(x), PyExc_BaseException))
diff --git a/pypy/module/cpyext/include/tupleobject.h b/pypy/module/cpyext/include/tupleobject.h
--- a/pypy/module/cpyext/include/tupleobject.h
+++ b/pypy/module/cpyext/include/tupleobject.h
@@ -26,6 +26,9 @@
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+#define PyTuple_Check(op) \
+		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_TUPLE_SUBCLASS)
+#define PyTuple_CheckExact(op) ((op)->ob_type == &PyTuple_Type)
 
 #ifdef __cplusplus
 }
diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -7,6 +7,11 @@
 
 #include "cpyext_unicodeobject.h"
 
+/*#define PyUnicode_Check(op) \
+**		 PyType_FastSubclass((op)->ob_type, Py_TPFLAGS_UNICODE_SUBCLASS)
+**#define PyUnicode_CheckExact(op) ((op)->ob_type == &PyUnicode_Type)
+*/
+
 /* Fast access macros */
 #ifndef Py_LIMITED_API
 
diff --git a/pypy/module/cpyext/listobject.py b/pypy/module/cpyext/listobject.py
--- a/pypy/module/cpyext/listobject.py
+++ b/pypy/module/cpyext/listobject.py
@@ -2,14 +2,14 @@
 from rpython.rlib.objectmodel import always_inline
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (cpython_api, CANNOT_FAIL, Py_ssize_t,
-                                    build_type_checkers)
+                                    build_type_checkers_flags)
 from pypy.module.cpyext.pyerrors import PyErr_BadInternalCall
 from pypy.module.cpyext.pyobject import decref, incref, PyObject, make_ref
 from pypy.objspace.std.listobject import W_ListObject
 from pypy.interpreter.error import oefmt
 
 
-PyList_Check, PyList_CheckExact = build_type_checkers("List")
+PyList_Check, PyList_CheckExact = build_type_checkers_flags("List")
 
 @cpython_api([Py_ssize_t], PyObject)
 def PyList_New(space, len):
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -1,13 +1,13 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.module.cpyext.api import (
-    cpython_api, PyObject, build_type_checkers, Py_ssize_t,
+    cpython_api, PyObject, build_type_checkers_flags, Py_ssize_t,
     CONST_STRING, ADDR, CANNOT_FAIL)
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rbigint import rbigint
 
 
-PyLong_Check, PyLong_CheckExact = build_type_checkers("Long", "w_int")
+PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long", "w_int")
 
 @cpython_api([lltype.Signed], PyObject)
 def PyLong_FromLong(space, val):
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    build_type_checkers, cpython_api, generic_cpy_call, CANNOT_FAIL,
+    cpython_api, generic_cpy_call, CANNOT_FAIL,
     PyTypeObjectPtr, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -1,3 +1,4 @@
+#pragma once
 
 typedef long Py_ssize_t;
 
diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py
--- a/pypy/module/cpyext/pytraceback.py
+++ b/pypy/module/cpyext/pytraceback.py
@@ -1,7 +1,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     PyObjectFields, generic_cpy_call, CONST_STRING, CANNOT_FAIL, Py_ssize_t,
-    cpython_api, bootstrap_function, cpython_struct, build_type_checkers,
+    cpython_api, bootstrap_function, cpython_struct, 
     slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, Py_DecRef, make_typedescr)
diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -147,53 +147,6 @@
         ten = foo.test_buffer(arr)
         assert ten == 10
 
-    @pytest.mark.skipif(True, reason="no _numpypy on py3k")
-    #@pytest.mark.skipif(only_pypy, reason='pypy only test')
-    def test_buffer_info(self):
-        try:
-            from _numpypy import multiarray as np
-        except ImportError:
-            skip('pypy built without _numpypy')
-        module = self.import_module(name='buffer_test')
-        get_buffer_info = module.get_buffer_info
-        raises(ValueError, get_buffer_info, np.arange(5)[::2], ('SIMPLE',))
-        arr = np.zeros((1, 10), order='F')
-        shape, strides = get_buffer_info(arr, ['F_CONTIGUOUS'])
-        assert strides[0] == 8
-        arr = np.zeros((10, 1), order='C')
-        shape, strides = get_buffer_info(arr, ['C_CONTIGUOUS'])
-        assert strides[-1] == 8
-        dt1 = np.dtype(
-             [('a', 'b'), ('b', 'i'),
-              ('sub0', np.dtype('b,i')),
-              ('sub1', np.dtype('b,i')),
-              ('sub2', np.dtype('b,i')),
-              ('sub3', np.dtype('b,i')),
-              ('sub4', np.dtype('b,i')),
-              ('sub5', np.dtype('b,i')),
-              ('sub6', np.dtype('b,i')),
-              ('sub7', np.dtype('b,i')),
-              ('c', 'i')],
-             )
-        x = np.arange(dt1.itemsize, dtype='int8').view(dt1)
-        # pytest can catch warnings from v2.8 and up, we ship 2.5
-        import warnings
-        warnings.filterwarnings("error")
-        try:
-            try:
-                y = get_buffer_info(x, ['SIMPLE'])
-            except UserWarning as e:
-                pass
-            else:
-                assert False ,"PyPy-specific UserWarning not raised" \
-                          " on too long format string"
-        finally:
-            warnings.resetwarnings()
-        # calling get_buffer_info on x creates a memory leak,
-        # which is detected as an error at test teardown:
-        # Exception TypeError: "'NoneType' object is not callable"
-        #         in <bound method ConcreteArray.__del__ ...> ignored
-
     def test_releasebuffer(self):
         module = self.import_extension('foo', [
             ("create_test", "METH_NOARGS",
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -28,6 +28,8 @@
 NULL = lltype.nullptr(rffi.VOIDP.TO)
 
 class TestNDArrayObject(BaseApiTest):
+    spaceconfig = AppTestCpythonExtensionBase.spaceconfig.copy()
+    spaceconfig['usemodules'].append('micronumpy')
 
     def test_Check(self, space, api):
         a = array(space, [10, 5, 3])
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1049,7 +1049,7 @@
 
     def test_call_tp_dealloc(self):
         module = self.import_extension('foo', [
-            ("fetchFooType", "METH_VARARGS",
+            ("fetchFooType", "METH_NOARGS",
              """
                 PyObject *o;
                 o = PyObject_New(PyObject, &Foo_Type);
@@ -1067,7 +1067,7 @@
                 Py_DECREF(e);
                 return o;
              """),
-            ("getCounter", "METH_VARARGS",
+            ("getCounter", "METH_NOARGS",
              """
                 return PyLong_FromLong(foo_counter);
              """)], prologue="""
@@ -1334,3 +1334,50 @@
         # this is equivalent to
         from collections import Hashable
         assert not isinstance(obj, Hashable)
+
+
+class AppTestFlags(AppTestCpythonExtensionBase):
+    def test_has_subclass_flag(self):
+        module = self.import_extension('foo', [
+           ("test_flags", "METH_VARARGS",
+            '''
+                long in_flag, my_flag;
+                PyObject * obj;
+                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                    return NULL;
+                if (!PyType_Check(obj))
+                {
+                    PyErr_SetString(PyExc_ValueError, "input must be type");
+                    return NULL;
+                }
+                my_flag = ((PyTypeObject*)obj)->tp_flags;
+                if ((my_flag & in_flag) != in_flag)
+                    return PyLong_FromLong(-1);
+                if (!PyType_CheckExact(obj)) {
+                    if ((my_flag & Py_TPFLAGS_TYPE_SUBCLASS) == Py_TPFLAGS_TYPE_SUBCLASS)
+                        return PyLong_FromLong(-2);
+                }
+                return PyLong_FromLong(0);
+            '''),])
+        # copied from object.h
+        Py_TPFLAGS_LONG_SUBCLASS = (1<<24)
+        Py_TPFLAGS_LIST_SUBCLASS = (1<<25)
+        Py_TPFLAGS_TUPLE_SUBCLASS = (1<<26)
+        Py_TPFLAGS_BYTES_SUBCLASS = (1<<27)
+        Py_TPFLAGS_UNICODE_SUBCLASS = (1<<28)
+        Py_TPFLAGS_DICT_SUBCLASS = (1<<29)
+        Py_TPFLAGS_BASE_EXC_SUBCLASS = (1<<30)
+        Py_TPFLAGS_TYPE_SUBCLASS = (1<<31)
+        for t,f in ((int, Py_TPFLAGS_LONG_SUBCLASS),
+                    (list, Py_TPFLAGS_LIST_SUBCLASS),
+                    (tuple, Py_TPFLAGS_TUPLE_SUBCLASS),
+                    (bytes, Py_TPFLAGS_BYTES_SUBCLASS),
+                    (str, Py_TPFLAGS_UNICODE_SUBCLASS),
+                    (dict, Py_TPFLAGS_DICT_SUBCLASS),
+                    (Exception, Py_TPFLAGS_BASE_EXC_SUBCLASS),
+                    (type, Py_TPFLAGS_TYPE_SUBCLASS),
+                   ):
+            assert module.test_flags(t, f) == 0
+        class MyList(list):
+            pass
+        assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0
diff --git a/pypy/module/cpyext/tupleobject.py b/pypy/module/cpyext/tupleobject.py
--- a/pypy/module/cpyext/tupleobject.py
+++ b/pypy/module/cpyext/tupleobject.py
@@ -2,7 +2,7 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import fatalerror_notb
 from pypy.module.cpyext.api import (
-    cpython_api, Py_ssize_t, build_type_checkers,
+    cpython_api, Py_ssize_t, build_type_checkers_flags,
     PyVarObjectFields, cpython_struct, bootstrap_function, slot_function)
 from pypy.module.cpyext.pyobject import (
     PyObject, PyObjectP, make_ref, from_ref, decref, incref,
@@ -42,7 +42,7 @@
                    dealloc=tuple_dealloc,
                    realize=tuple_realize)
 
-PyTuple_Check, PyTuple_CheckExact = build_type_checkers("Tuple")
+PyTuple_Check, PyTuple_CheckExact = build_type_checkers_flags("Tuple")
 
 def tuple_check_ref(space, ref):
     w_type = from_ref(space, rffi.cast(PyObject, ref.c_ob_type))
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,23 +1,25 @@
-import os
-
 from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
 
-from pypy.interpreter.baseobjspace import W_Root, DescrMismatch
+from pypy.interpreter.baseobjspace import DescrMismatch
 from pypy.interpreter.error import oefmt
-from pypy.interpreter.typedef import (GetSetProperty, TypeDef,
-        interp_attrproperty, interp2app)
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, interp_attrproperty, interp2app)
 from pypy.module.__builtin__.abstractinst import abstract_issubclass_w
 from pypy.module.cpyext import structmemberdefs
 from pypy.module.cpyext.api import (
-    cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
-    slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
-    Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers, Py_TPFLAGS_BASETYPE,
-    PyObjectFields, PyTypeObject, PyTypeObjectPtr,
-    cts, parse_dir)
+    cpython_api, cpython_struct, bootstrap_function, Py_ssize_t,
+    slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
+    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
+    PyTypeObjectPtr, Py_buffer,
+    Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
+    Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
+    Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
+    Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
+    Py_TPFLAGS_TYPE_SUBCLASS,
+    Py_TPFLAGS_BYTES_SUBCLASS)
 from pypy.module.cpyext.cparser import CTypeSpace
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -32,14 +34,14 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.structmember import PyMember_GetOne, PyMember_SetOne
 from pypy.module.cpyext.typeobjectdefs import (
-    PyGetSetDef, PyMemberDef, PyMappingMethods, getter, setter,
+    PyGetSetDef, PyMemberDef, PyMappingMethods,
     PyNumberMethods, PySequenceMethods, PyBufferProcs)
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
-PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
+PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
@@ -431,7 +433,7 @@
     dict_w["__new__"] = PyCFunction_NewEx(space, get_new_method_def(space),
                                           from_ref(space, pyo), None)
 
-def inherit_special(space, pto, base_pto):
+def inherit_special(space, pto, w_obj, base_pto):
     # XXX missing: copy basicsize and flags in a magical way
     # (minimally, if tp_basicsize is zero or too low, we copy it from the base)
     if pto.c_tp_basicsize < base_pto.c_tp_basicsize:
@@ -439,6 +441,24 @@
     if pto.c_tp_itemsize < base_pto.c_tp_itemsize:
         pto.c_tp_itemsize = base_pto.c_tp_itemsize
 
+    #/* Setup fast subclass flags */
+    if space.issubtype_w(w_obj, space.w_BaseException):
+        pto.c_tp_flags |= Py_TPFLAGS_BASE_EXC_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_type):
+        pto.c_tp_flags |= Py_TPFLAGS_TYPE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_int):
+        pto.c_tp_flags |= Py_TPFLAGS_LONG_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_bytes):
+        pto.c_tp_flags |= Py_TPFLAGS_BYTES_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_unicode):
+        pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_tuple):
+        pto.c_tp_flags |= Py_TPFLAGS_TUPLE_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_list):
+        pto.c_tp_flags |= Py_TPFLAGS_LIST_SUBCLASS
+    elif space.issubtype_w(w_obj, space.w_dict):
+        pto.c_tp_flags |= Py_TPFLAGS_DICT_SUBCLASS
+
 def check_descr(space, w_self, w_type):
     if not space.isinstance_w(w_self, w_type):
         raise DescrMismatch()
@@ -845,7 +865,7 @@
     pto.c_tp_mro = make_ref(space, space.newtuple(w_obj.mro_w))
     base = pto.c_tp_base
     if base:
-        inherit_special(space, pto, base)
+        inherit_special(space, pto, w_obj, base)
     for w_base in space.fixedview(from_ref(space, pto.c_tp_bases)):
         if isinstance(w_base, W_TypeObject):
             inherit_slots(space, pto, w_base)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -38,7 +38,7 @@
 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
                                  flavor='raw', zero=True)
 
-PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode", "w_unicode")
+PyUnicode_Check, PyUnicode_CheckExact = build_type_checkers("Unicode")
 
 MAX_UNICODE = 1114111
 WCHAR_KIND = 0
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -2,7 +2,7 @@
 import pytest
 
 
-class AppTestItertools:
+class AppTestItertools(object):
     spaceconfig = dict(usemodules=['itertools'])
 
     def test_count(self):
@@ -646,7 +646,7 @@
         assert itertools.tee(a, 0) == ()
 
 
-class AppTestItertools26:
+class AppTestItertools26(object):
     spaceconfig = dict(usemodules=['itertools'])
 
     def test_count_overflow(self):
@@ -908,10 +908,8 @@
         raises(ValueError, permutations, [1, 2], -1)
 
 
-class AppTestItertools27:
-    spaceconfig = {
-        "usemodules": ['itertools', 'struct', 'binascii'],
-    }
+class AppTestItertools27(object):
+    spaceconfig = {"usemodules": ['itertools', 'struct', 'binascii']}
 
     def test_compress(self):
         import itertools
diff --git a/pypy/module/pyexpat/__init__.py b/pypy/module/pyexpat/__init__.py
--- a/pypy/module/pyexpat/__init__.py
+++ b/pypy/module/pyexpat/__init__.py
@@ -65,6 +65,14 @@
                  'XML_PARAM_ENTITY_PARSING_ALWAYS']:
         interpleveldefs[name] = 'space.newint(interp_pyexpat.%s)' % (name,)
 
+    def __init__(self, space, w_name):
+        "NOT_RPYTHON"
+        from pypy.module.pyexpat import interp_pyexpat
+        super(Module, self).__init__(space, w_name)
+        ver = space.unwrap(interp_pyexpat.get_expat_version(space))
+        assert len(ver) >= 5, (
+            "Cannot compile with the wide (UTF-16) version of Expat")
+
     def startup(self, space):
         from pypy.module.pyexpat import interp_pyexpat
         w_ver = interp_pyexpat.get_expat_version(space)
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -120,18 +120,6 @@
         locals()[name] = rffi_platform.ConstantInteger(name)
     for name in xml_model_list:
         locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
-    for name in xml_model_list:
-        locals()[name] = rffi_platform.ConstantInteger(name)
     XML_Parser_SIZE = rffi_platform.SizeOf("XML_Parser")
 
 for k, v in rffi_platform.configure(CConfigure).items():
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -115,10 +115,10 @@
     if check_int_or_float:
         for w_obj in list_w:
             if type(w_obj) is W_IntObject:
-                if longlong2float.can_encode_int32(space.int_w(w_obj)):
+                if longlong2float.can_encode_int32(w_obj.int_w(space)):
                     continue    # ok
             elif type(w_obj) is W_FloatObject:
-                if longlong2float.can_encode_float(space.float_w(w_obj)):
+                if longlong2float.can_encode_float(w_obj.float_w(space)):
                     continue    # ok
             break
         else:
diff --git a/pypy/objspace/std/specialisedtupleobject.py b/pypy/objspace/std/specialisedtupleobject.py
--- a/pypy/objspace/std/specialisedtupleobject.py
+++ b/pypy/objspace/std/specialisedtupleobject.py
@@ -31,23 +31,23 @@
     class cls(W_AbstractTupleObject):
         _immutable_fields_ = ['value%s' % i for i in iter_n]
 
-        def __init__(self, space, *values_w):
+        def __init__(self, space, *values):
             self.space = space
-            assert len(values_w) == typelen
+            assert len(values) == typelen
             for i in iter_n:
-                w_obj = values_w[i]
+                obj = values[i]
                 val_type = typetuple[i]
                 if val_type == int:
-                    unwrapped = w_obj.int_w(space)
+                    assert isinstance(obj, int)
                 elif val_type == float:
-                    unwrapped = w_obj.float_w(space)
+                    assert isinstance(obj, float)
                 elif val_type == str:
-                    unwrapped = w_obj.str_w(space)
+                    assert isinstance(obj, str)
                 elif val_type == object:
-                    unwrapped = w_obj
+                    pass
                 else:
                     raise AssertionError
-                setattr(self, 'value%s' % i, unwrapped)
+                setattr(self, 'value%s' % i, obj)
 
         def length(self):
             return typelen
@@ -155,10 +155,10 @@
         w_arg1, w_arg2 = list_w
         if type(w_arg1) is W_IntObject:
             if type(w_arg2) is W_IntObject:
-                return Cls_ii(space, w_arg1, w_arg2)
+                return Cls_ii(space, space.int_w(w_arg1), space.int_w(w_arg2))
         elif type(w_arg1) is W_FloatObject:
             if type(w_arg2) is W_FloatObject:
-                return Cls_ff(space, w_arg1, w_arg2)
+                return Cls_ff(space, space.float_w(w_arg1), space.float_w(w_arg2))
         return Cls_oo(space, w_arg1, w_arg2)
     else:
         raise NotSpecialised
@@ -175,10 +175,9 @@
 # faster to move the decision out of the loop.
 
 @specialize.arg(1)
-def _build_zipped_spec(space, Cls, lst1, lst2, wrap1, wrap2):
+def _build_zipped_spec(space, Cls, lst1, lst2):
     length = min(len(lst1), len(lst2))
-    return [Cls(space, wrap1(lst1[i]),
-                       wrap2(lst2[i])) for i in range(length)]
+    return [Cls(space, lst1[i], lst2[i]) for i in range(length)]
 
 def _build_zipped_spec_oo(space, w_list1, w_list2):
     strat1 = w_list1.strategy
@@ -205,8 +204,7 @@
             intlist2 = w_list2.getitems_int()
             if intlist2 is not None:
                 lst_w = _build_zipped_spec(
-                        space, Cls_ii, intlist1, intlist2,
-                        space.newint, space.newint)
+                        space, Cls_ii, intlist1, intlist2)
                 return space.newlist(lst_w)
         else:
             floatlist1 = w_list1.getitems_float()
@@ -214,8 +212,7 @@
                 floatlist2 = w_list2.getitems_float()
                 if floatlist2 is not None:
                     lst_w = _build_zipped_spec(
-                        space, Cls_ff, floatlist1, floatlist2, space.newfloat,
-                        space.newfloat)
+                        space, Cls_ff, floatlist1, floatlist2)
                     return space.newlist(lst_w)
 
         lst_w = _build_zipped_spec_oo(space, w_list1, w_list2)
diff --git a/rpython/annotator/test/test_annrpython.py b/rpython/annotator/test/test_annrpython.py
--- a/rpython/annotator/test/test_annrpython.py
+++ b/rpython/annotator/test/test_annrpython.py
@@ -4644,6 +4644,25 @@
         s_exc = a.binding(graphof(a, f).exceptblock.inputargs[1])
         assert not s_exc.can_be_none()
 
+    def test_specialize_argtype_with_subclasses(self):
+        # checks that specialize:argtype() makes two copies of a
+        # function f(), one for the base class and one for the subclass
+        class A:
+            def foo(self):
+                return 123
+        class B(A):
+            def foo(self):
+                return 456
+        def f(x):
+            return x.foo()
+        f._annspecialcase_ = "specialize:argtype(0)"
+        def h(y):
+            if y > 5:
+                f(A())
+            return f(B())
+        a = self.RPythonAnnotator()
+        assert a.build_types(h, [int]).const == 456
+
 
 def g(n):
     return [0, 1, 2, n]
diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py
--- a/rpython/jit/backend/llsupport/regalloc.py
+++ b/rpython/jit/backend/llsupport/regalloc.py
@@ -10,6 +10,10 @@
 except ImportError:
     OrderedDict = dict # too bad
 
+SAVE_DEFAULT_REGS = 0
+SAVE_GCREF_REGS = 2
+SAVE_ALL_REGS = 1
+
 class TempVar(AbstractValue):
     def __init__(self):
         pass
@@ -586,7 +590,8 @@
                                                  force_store, save_all_regs)
 
     def spill_or_move_registers_before_call(self, save_sublist,
-                                            force_store=[], save_all_regs=0):
+                                            force_store=[],
+                                            save_all_regs=SAVE_DEFAULT_REGS):
         """Spill or move some registers before a call.
 
         By default, this means: for every register in 'save_sublist',
@@ -602,8 +607,9 @@
         valid, but only *if they are in self.save_around_call_regs,*
         not if they are callee-saved registers!
 
-        'save_all_regs' can be 0 (default set of registers), 1 (do that
-        for all registers), or 2 (default + gc ptrs).
+        'save_all_regs' can be SAVE_DEFAULT_REGS (default set of registers),
+        SAVE_ALL_REGS (do that for all registers), or SAVE_GCREF_REGS (default
+        + gc ptrs).
 
         Overview of what we do (the implementation does it differently,
         for the same result):
@@ -651,11 +657,11 @@
                 new_free_regs.append(reg)
                 continue
 
-            if save_all_regs == 1:
+            if save_all_regs == SAVE_ALL_REGS:
                 # we need to spill all registers in this mode
                 self._bc_spill(v, new_free_regs)
                 #
-            elif save_all_regs == 2 and v.type == REF:
+            elif save_all_regs == SAVE_GCREF_REGS and v.type == REF:
                 # we need to spill all GC ptrs in this mode
                 self._bc_spill(v, new_free_regs)
                 #
diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -2,13 +2,13 @@
 """ Register allocation scheme.
 """
 
-import os, sys
 from rpython.jit.backend.llsupport import symbolic
 from rpython.jit.backend.llsupport.descr import CallDescr, unpack_arraydescr
 from rpython.jit.backend.llsupport.gcmap import allocate_gcmap
 from rpython.jit.backend.llsupport.regalloc import (FrameManager, BaseRegalloc,
      RegisterManager, TempVar, compute_vars_longevity, is_comparison_or_ovf_op,
-     valid_addressing_size, get_scale)
+     valid_addressing_size, get_scale, SAVE_DEFAULT_REGS, SAVE_GCREF_REGS,
+     SAVE_ALL_REGS)
 from rpython.jit.backend.x86 import rx86
 from rpython.jit.backend.x86.arch import (WORD, JITFRAME_FIXED_SIZE, IS_X86_32,
     IS_X86_64, DEFAULT_FRAME_BYTES)
@@ -23,12 +23,11 @@
 from rpython.jit.codewriter.effectinfo import EffectInfo
 from rpython.jit.metainterp.history import (Const, ConstInt, ConstPtr,
     ConstFloat, INT, REF, FLOAT, VECTOR, TargetToken, AbstractFailDescr)
-from rpython.jit.metainterp.resoperation import rop, ResOperation
+from rpython.jit.metainterp.resoperation import rop
 from rpython.jit.metainterp.resume import AccumInfo
 from rpython.rlib import rgc
 from rpython.rlib.objectmodel import we_are_translated
 from rpython.rlib.rarithmetic import r_longlong, r_uint
-from rpython.rtyper.annlowlevel import cast_instance_to_gcref
 from rpython.rtyper.lltypesystem import lltype, rffi, rstr
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.jit.backend.x86.regloc import AddressLoc
@@ -799,25 +798,29 @@
         # we need to save registers on the stack:
         #
         #  - at least the non-callee-saved registers
+        #    (gc_level == SAVE_DEFAULT_REGS)
         #
-        #  - if gc_level > 0, we save also the callee-saved registers that
-        #    contain GC pointers
+        #  - if gc_level == SAVE_GCREF_REGS we save also the callee-saved
+        #    registers that contain GC pointers
         #
-        #  - gc_level == 2 for CALL_MAY_FORCE or CALL_ASSEMBLER.  We
+        #  - gc_level == SAVE_ALL_REGS for CALL_MAY_FORCE or CALL_ASSEMBLER.  We
         #    have to save all regs anyway, in case we need to do
         #    cpu.force().  The issue is that grab_frame_values() would
         #    not be able to locate values in callee-saved registers.
         #
-        save_all_regs = gc_level == 2
+        if gc_level == SAVE_ALL_REGS:
+            save_all_regs = SAVE_ALL_REGS
+        else:
+            save_all_regs = SAVE_DEFAULT_REGS
         self.xrm.before_call(save_all_regs=save_all_regs)
-        if gc_level == 1:
+        if gc_level == SAVE_GCREF_REGS:
             gcrootmap = self.assembler.cpu.gc_ll_descr.gcrootmap
-            # we save all the registers for shadowstack and asmgcc for now
+            # we save all the GCREF registers for shadowstack and asmgcc for now
             # --- for asmgcc too: we can't say "register x is a gc ref"
             # without distinguishing call sites, which we don't do any
             # more for now.
             if gcrootmap: # and gcrootmap.is_shadow_stack:
-                save_all_regs = 2
+                save_all_regs = SAVE_GCREF_REGS
         self.rm.before_call(save_all_regs=save_all_regs)
         if op.type != 'v':
             if op.type == FLOAT:
@@ -841,11 +844,11 @@
         #
         effectinfo = calldescr.get_extra_info()
         if guard_not_forced:
-            gc_level = 2
+            gc_level = SAVE_ALL_REGS
         elif effectinfo is None or effectinfo.check_can_collect():
-            gc_level = 1
+            gc_level = SAVE_GCREF_REGS
         else:
-            gc_level = 0
+            gc_level = SAVE_DEFAULT_REGS
         #
         self._call(op, [imm(size), sign_loc] +
                        [self.loc(op.getarg(i)) for i in range(op.numargs())],
@@ -909,7 +912,7 @@
 
     def _consider_call_assembler(self, op):
         locs = self.locs_for_call_assembler(op)
-        self._call(op, locs, gc_level=2)
+        self._call(op, locs, gc_level=SAVE_ALL_REGS)
     consider_call_assembler_i = _consider_call_assembler
     consider_call_assembler_r = _consider_call_assembler
     consider_call_assembler_f = _consider_call_assembler
diff --git a/rpython/jit/codewriter/call.py b/rpython/jit/codewriter/call.py
--- a/rpython/jit/codewriter/call.py
+++ b/rpython/jit/codewriter/call.py
@@ -185,8 +185,30 @@
                                          FUNC.RESULT, EffectInfo.MOST_GENERAL)
         return (fnaddr, calldescr)
 
+    def _raise_effect_error(self, op, extraeffect, functype, calling_graph):
+        explanation = []
+        if extraeffect == EffectInfo.EF_RANDOM_EFFECTS:
+            explanation = self.randomeffects_analyzer.explain_analyze_slowly(op)
+        elif extraeffect == EffectInfo.EF_FORCES_VIRTUAL_OR_VIRTUALIZABLE:
+            explanation = self.virtualizable_analyzer.explain_analyze_slowly(op)
+        msg = []
+        if explanation:
+            msg = [
+                "_______ ERROR AT BOTTOM ______",
+                "RPython callstack leading to problem:",
+                ]
+            msg.extend(explanation)
+            msg.append("_______ ERROR: ______")
+        msg.append("operation %r" % op)
+        msg.append("in graph %s" % (calling_graph or "<unknown>"))
+        msg.append("this calls a %s function," % (functype, ))
+        msg.append(" but this contradicts other sources (e.g. it can have random"
+                   " effects): EF=%s" % (extraeffect, ))
+        raise Exception("\n".join(msg))
+
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None,
+                     calling_graph=None):
         """Return the calldescr that describes all calls done by 'op'.
         This returns a calldescr that we can put in the corresponding
         call operation in the calling jitcode.  It gets an effectinfo
@@ -202,13 +224,13 @@
         ARGS = FUNC.ARGS
         if NON_VOID_ARGS != [T for T in ARGS if T is not lltype.Void]:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
+                "operation %r in %s: calling a function with signature %r, "
                 "but passing actual arguments (ignoring voids) of types %r"
-                % (op, FUNC, NON_VOID_ARGS))
+                % (op, calling_graph, FUNC, NON_VOID_ARGS))
         if RESULT != FUNC.RESULT:
             raise Exception(
-                "in operation %r: calling a function with signature %r, "
-                "but the actual return type is %r" % (op, FUNC, RESULT))
+                "%r in %s: calling a function with signature %r, "
+                "but the actual return type is %r" % (op, calling_graph, FUNC, RESULT))
         # ok
         # get the 'elidable' and 'loopinvariant' flags from the function object
         elidable = False
@@ -217,7 +239,7 @@
         if op.opname == "direct_call":
             funcobj = op.args[0].value._obj
             assert getattr(funcobj, 'calling_conv', 'c') == 'c', (
-                "%r: getcalldescr() with a non-default call ABI" % (op,))
+                "%r in %s: getcalldescr() with a non-default call ABI" % (op, calling_graph))
             func = getattr(funcobj, '_callable', None)
             elidable = getattr(func, "_elidable_function_", False)
             loopinvariant = getattr(func, "_jit_loop_invariant_", False)
@@ -245,11 +267,11 @@
                 if not error:
                     continue
                 raise Exception(
-                    "%r is an indirect call to a family of functions "
+                    "%r in %s is an indirect call to a family of functions "
                     "(or methods) that includes %r. However, the latter "
                     "is marked %r. You need to use an indirection: replace "
                     "it with a non-marked function/method which calls the "
-                    "marked function." % (op, graph, error))
+                    "marked function." % (op, calling_graph, graph, error))
         # build the extraeffect
         random_effects = self.randomeffects_analyzer.analyze(op)
         if random_effects:
@@ -277,22 +299,17 @@
         # check that the result is really as expected
         if loopinvariant:
             if extraeffect != EffectInfo.EF_LOOPINVARIANT:
-                raise Exception(
-                "in operation %r: this calls a _jit_loop_invariant_ function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+                self._raise_effect_error(op, extraeffect, "_jit_loop_invariant_", calling_graph)
         if elidable:
             if extraeffect not in (EffectInfo.EF_ELIDABLE_CANNOT_RAISE,
                                    EffectInfo.EF_ELIDABLE_OR_MEMORYERROR,
                                    EffectInfo.EF_ELIDABLE_CAN_RAISE):
-                raise Exception(
-                "in operation %r: this calls an elidable function,"
-                " but this contradicts other sources (e.g. it can have random"
-                " effects): EF=%s" % (op, extraeffect))
+
+                self._raise_effect_error(op, extraeffect, "elidable", calling_graph)
             elif RESULT is lltype.Void:
                 raise Exception(
-                    "in operation %r: this calls an elidable function "
-                    "but the function has no result" % (op, ))
+                    "operation %r in %s: this calls an elidable function "
+                    "but the function has no result" % (op, calling_graph))
         #
         effectinfo = effectinfo_from_writeanalyze(
             self.readwrite_analyzer.analyze(op, self.seen_rw), self.cpu,
diff --git a/rpython/jit/codewriter/jtransform.py b/rpython/jit/codewriter/jtransform.py
--- a/rpython/jit/codewriter/jtransform.py
+++ b/rpython/jit/codewriter/jtransform.py
@@ -64,6 +64,7 @@
         self.cpu = cpu
         self.callcontrol = callcontrol
         self.portal_jd = portal_jd   # non-None only for the portal graph(s)
+        self.graph = None
 
     def transform(self, graph):
         self.graph = graph
@@ -424,7 +425,8 @@
         of 'residual_call_xxx' are the function to call, and its calldescr."""
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex=oopspecindex,
                                                   extraeffect=extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         op1 = self.rewrite_call(op, 'residual_call',
                                 [op.args[0]] + extraargs, calldescr=calldescr)
         if may_call_jitcodes or self.callcontrol.calldescr_canraise(calldescr):
@@ -1613,7 +1615,9 @@
         if len(op.args) > 4 + 2 or have_floats:
             raise Exception("Conditional call does not support floats or more than 4 arguments")
         callop = SpaceOperation('direct_call', op.args[1:], op.result)
-        calldescr = self.callcontrol.getcalldescr(callop)
+        calldescr = self.callcontrol.getcalldescr(
+                callop,
+                calling_graph=self.graph)
         assert not calldescr.get_extra_info().check_forces_virtual_or_virtualizable()
         op1 = self.rewrite_call(op, rewritten_opname,
                                 op.args[:2], args=op.args[2:],
@@ -1924,7 +1928,8 @@
                              extradescr=None):
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
                                                   extraeffect,
-                                                  extradescr=extradescr)
+                                                  extradescr=extradescr,
+                                                  calling_graph=self.graph)
         if extraeffect is not None:
             assert (is_test_calldescr(calldescr)      # for tests
                     or calldescr.get_extra_info().extraeffect == extraeffect)
@@ -1954,7 +1959,8 @@
                             [c_func] + [varoftype(T) for T in argtypes],
                             varoftype(resulttype))
         calldescr = self.callcontrol.getcalldescr(op, oopspecindex,
-                                                  effectinfo)
+                                                  effectinfo,
+                                                  calling_graph=self.graph)
         if isinstance(c_func.value, str):    # in tests only
             func = c_func.value
         else:
diff --git a/rpython/jit/codewriter/test/test_call.py b/rpython/jit/codewriter/test/test_call.py
--- a/rpython/jit/codewriter/test/test_call.py
+++ b/rpython/jit/codewriter/test/test_call.py
@@ -299,11 +299,23 @@
     def f4(n, m):
         return compute_hash(str(n) + str(m))
 
+    T = rffi.CArrayPtr(rffi.TIME_T)
+    external = rffi.llexternal("time", [T], rffi.TIME_T, releasegil=True)
+
+    def effect():
+        return external(lltype.nullptr(T.TO))
+
+    @jit.elidable
+    def f5(n, m):
+        effect()
+        return 1
+
     def f(n, m):
         a = f1(n, m)
         b = f2(n, m)
         c = f3(n, m)
         d = f4(n, m)
+        f5(n, m)
         enable_siphash24()
         return a + len(b) + c + d
 
@@ -323,25 +335,33 @@
         call_descr = cc.getcalldescr(call_op)
         assert call_descr.extrainfo.extraeffect == expected
 
+    call_op = f_graph.startblock.operations[4]
+    assert call_op.opname == 'direct_call'
+    excinfo = py.test.raises(Exception, cc.getcalldescr, call_op)
+    lines = excinfo.value.args[0].splitlines()
+    assert "f5" in lines[2]
+    assert "effect" in lines[3]
+    assert "random effects" in lines[-1]
+
 def test_raise_elidable_no_result():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
     l = []
     @jit.elidable
     def f1(n, m):
         l.append(n)
-    def f(n, m):
+    def fancy_graph_name(n, m):
         f1(n, m)
         return n + m
 
-    rtyper = support.annotate(f, [7, 9])
+    rtyper = support.annotate(fancy_graph_name, [7, 9])
     jitdriver_sd = FakeJitDriverSD(rtyper.annotator.translator.graphs[0])
     cc = CallControl(LLGraphCPU(rtyper), jitdrivers_sd=[jitdriver_sd])
     res = cc.find_all_graphs(FakePolicy())
-    [f_graph] = [x for x in res if x.func is f]
+    [f_graph] = [x for x in res if x.func is fancy_graph_name]
     call_op = f_graph.startblock.operations[0]
     assert call_op.opname == 'direct_call'
-    with py.test.raises(Exception):
-        call_descr = cc.getcalldescr(call_op)
+    x = py.test.raises(Exception, cc.getcalldescr, call_op, calling_graph=f_graph)
+    assert "fancy_graph_name" in str(x.value)
 
 def test_can_or_cannot_collect():
     from rpython.jit.backend.llgraph.runner import LLGraphCPU
diff --git a/rpython/jit/codewriter/test/test_flatten.py b/rpython/jit/codewriter/test/test_flatten.py
--- a/rpython/jit/codewriter/test/test_flatten.py
+++ b/rpython/jit/codewriter/test/test_flatten.py
@@ -73,7 +73,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=EffectInfo.OS_NONE,
-                     extraeffect=None, extradescr=None):
+                     extraeffect=None, extradescr=None, calling_graph=None):
         try:
             name = op.args[0].value._obj._name
             if 'cannot_raise' in name or name.startswith('cast_'):
diff --git a/rpython/jit/codewriter/test/test_jtransform.py b/rpython/jit/codewriter/test/test_jtransform.py
--- a/rpython/jit/codewriter/test/test_jtransform.py
+++ b/rpython/jit/codewriter/test/test_jtransform.py
@@ -48,7 +48,7 @@
     def guess_call_kind(self, op):
         return 'residual'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         return 'calldescr'
     def calldescr_canraise(self, calldescr):
         return True
@@ -106,7 +106,7 @@
     def guess_call_kind(self, op):
         return 'builtin'
     def getcalldescr(self, op, oopspecindex=None, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
         assert oopspecindex is not None    # in this test
         EI = effectinfo.EffectInfo
         if oopspecindex != EI.OS_ARRAYCOPY:
diff --git a/rpython/jit/codewriter/test/test_list.py b/rpython/jit/codewriter/test/test_list.py
--- a/rpython/jit/codewriter/test/test_list.py
+++ b/rpython/jit/codewriter/test/test_list.py
@@ -39,7 +39,7 @@
 class FakeCallControl:
     class getcalldescr(AbstractDescr):
         def __init__(self, op, oopspecindex=0, extraeffect=None,
-                     extradescr=None):
+                     extradescr=None, calling_graph=None):
             self.op = op
             self.oopspecindex = oopspecindex
         def __repr__(self):
diff --git a/rpython/jit/codewriter/test/test_longlong.py b/rpython/jit/codewriter/test/test_longlong.py
--- a/rpython/jit/codewriter/test/test_longlong.py
+++ b/rpython/jit/codewriter/test/test_longlong.py
@@ -17,7 +17,7 @@
 class FakeBuiltinCallControl:
     def guess_call_kind(self, op):
         return 'builtin'
-    def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None):
+    def getcalldescr(self, op, oopspecindex=None, extraeffect=None, extradescr=None, **kwargs):
         assert oopspecindex is not None    # in this test
         return 'calldescr-%d' % oopspecindex
     def calldescr_canraise(self, calldescr):
diff --git a/rpython/jit/metainterp/optimizeopt/heap.py b/rpython/jit/metainterp/optimizeopt/heap.py
--- a/rpython/jit/metainterp/optimizeopt/heap.py
+++ b/rpython/jit/metainterp/optimizeopt/heap.py
@@ -445,13 +445,13 @@
             if effectinfo.check_readonly_descr_field(fielddescr):
                 cf.force_lazy_set(self, fielddescr)
             if effectinfo.check_write_descr_field(fielddescr):
+                cf.force_lazy_set(self, fielddescr, can_cache=False)
                 if fielddescr.is_always_pure():
                     continue
                 try:
                     del self.cached_dict_reads[fielddescr]
                 except KeyError:
                     pass
-                cf.force_lazy_set(self, fielddescr, can_cache=False)
         #
         for arraydescr, submap in self.cached_arrayitems.items():
             if effectinfo.check_readonly_descr_array(arraydescr):
diff --git a/rpython/jit/metainterp/optimizeopt/info.py b/rpython/jit/metainterp/optimizeopt/info.py
--- a/rpython/jit/metainterp/optimizeopt/info.py
+++ b/rpython/jit/metainterp/optimizeopt/info.py
@@ -329,11 +329,14 @@
 
     def make_guards(self, op, short, optimizer):
         if self._known_class is not None:
-            short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
+                short.append(ResOperation(rop.GUARD_NONNULL, [op]))
                 short.append(ResOperation(rop.GUARD_IS_OBJECT, [op]))
-            short.append(ResOperation(rop.GUARD_CLASS,
-                                      [op, self._known_class]))
+                short.append(ResOperation(rop.GUARD_CLASS,
+                                          [op, self._known_class]))
+            else:
+                short.append(ResOperation(rop.GUARD_NONNULL_CLASS,
+                    [op, self._known_class]))
         elif self.descr is not None:
             short.append(ResOperation(rop.GUARD_NONNULL, [op]))
             if not optimizer.cpu.remove_gctypeptr:
diff --git a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
--- a/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
+++ b/rpython/jit/metainterp/optimizeopt/test/test_optimizeopt.py
@@ -3562,6 +3562,27 @@
         """
         self.optimize_loop(ops, expected, expected_preamble=expected_preamble)
 
+    def test_residual_call_still_forces_immutable_writes_though(self):
+        ops = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        expected_preamble = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        expected = """
+        [p1]
+        setfield_gc(p1, 6, descr=valuedescr3)
+        i2 = call_i(5, descr=writevalue3descr)
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected, expected_preamble=expected_preamble)
+
     def test_residual_call_invalidate_some_caches(self):
         ops = """
         [p1, p2]
@@ -7541,6 +7562,33 @@
         """
         self.optimize_loop(ops, expected, expected_short=short)
 
+    def test_guards_before_getfields_in_short_preamble_removetypeptr(self, monkeypatch):
+        monkeypatch.setattr(self.cpu, "remove_gctypeptr", True)
+        ops = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p0)
+        """
+        expected = """
+        [p0, p1]
+        jump(p0, p1)
+        """
+        short = """
+        [p0]
+        guard_nonnull_class(p0, ConstClass(node_vtable)) []
+        p1 = getfield_gc_r(p0, descr=nextdescr)
+        guard_nonnull_class(p1, ConstClass(node_vtable)) []
+        p2 = getfield_gc_r(p1, descr=nextdescr)
+        guard_nonnull_class(p2, ConstClass(node_vtable)) []
+        jump(p1)
+        """
+        self.optimize_loop(ops, expected, expected_short=short)
+
+
     def test_forced_virtual_pure_getfield(self):
         ops = """
         [p0]
diff --git a/rpython/jit/metainterp/test/test_loop.py b/rpython/jit/metainterp/test/test_loop.py
--- a/rpython/jit/metainterp/test/test_loop.py
+++ b/rpython/jit/metainterp/test/test_loop.py
@@ -1114,5 +1114,40 @@
         # one guard_false got removed
         self.check_resops(guard_false=4, guard_true=5)
 
+    def test_heapcache_bug(self):
+        class W_Object(object):
+            _attrs_ = []
+        class W_Nil(W_Object):
+            _attrs_ = []
+        class W_Cons(W_Object):
+            _attrs_ = ['first', 'rest']
+            _immutable_fields_ = ['first', 'rest']
+            def __init__(self, v1, v2):
+                self.first = v1
+                self.rest = v2
+
+        def reverse(xs):
+            result = W_Nil()
+            while isinstance(xs, W_Cons):
+                result = W_Cons(xs.first, result)
+                xs = xs.rest
+            return result
+
+        driver = JitDriver(reds=['repetitions', 'v'], greens=['pc'],
+                       get_printable_location=lambda pc: str(pc))
+        def entry_point():
+            repetitions = 0
+            while repetitions < 10:
+                pc = 0
+                v = W_Nil()
+                while pc < 10:
+                    driver.jit_merge_point(v=v, repetitions=repetitions, pc=pc)
+                    v = reverse(W_Cons(pc + 1, W_Cons(pc + 2, W_Cons(pc + 3, W_Cons(pc + 4, W_Nil())))))
+                    pc = pc + 1
+                repetitions += 1
+        
+        self.meta_interp(entry_point, [])
+
+
 class TestLLtype(LoopTest, LLJitMixin):
     pass
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -985,11 +985,14 @@
                          ]
 
 
-    compilation_info = ExternalCompilationInfo(
+    compilation_info = eci.merge(ExternalCompilationInfo(
                                     includes=includes,
                                     separate_module_sources=separate_module_sources,
                                     post_include_bits=post_include_bits,
-                               )
+                               ))
+else:
+    compilation_info = eci
+
 
 if _WIN32:
     CConfig.WSAEVENT = platform.SimpleType('WSAEVENT', rffi.VOIDP)
diff --git a/rpython/translator/backendopt/graphanalyze.py b/rpython/translator/backendopt/graphanalyze.py
--- a/rpython/translator/backendopt/graphanalyze.py
+++ b/rpython/translator/backendopt/graphanalyze.py
@@ -4,6 +4,7 @@
 
 class GraphAnalyzer(object):
     verbose = False
+    explanation = None
 
     def __init__(self, translator):
         self.translator = translator
@@ -73,6 +74,20 @@
     def compute_graph_info(self, graph):
         return None
 
+    def explain_analyze_slowly(self, op):
+        # this is a hack! usually done before a crash
+        self.__init__(self.translator)
+        self.explanation = explanation = []
+        oldverbose = self.verbose
+        self.verbose = True
+        try:
+            self.analyze(op)
+        finally:
+            del self.explanation
+            self.verbose = oldverbose
+        explanation.reverse()
+        return explanation
+
     def analyze(self, op, seen=None, graphinfo=None):
         if op.opname == "direct_call":
             try:
@@ -113,7 +128,11 @@
         return x
 
     def dump_info(self, info):
-        print '[%s] %s' % (self.__class__.__name__, info)
+        st = '[%s] %s' % (self.__class__.__name__, info)
+        if self.explanation is not None:
+            self.explanation.append(st)
+        else:
+            print st
 
     def analyze_direct_call(self, graph, seen=None):
         if seen is None:
diff --git a/rpython/translator/backendopt/test/test_writeanalyze.py b/rpython/translator/backendopt/test/test_writeanalyze.py
--- a/rpython/translator/backendopt/test/test_writeanalyze.py
+++ b/rpython/translator/backendopt/test/test_writeanalyze.py
@@ -531,3 +531,37 @@
         typed_effects = self._analyze_graph(t, wa, typed_write)
         typed_effects = self._filter_reads(typed_effects)
         assert typed_effects == direct_effects
+
+    def test_explanation(self):
+        class A(object):
+            def methodname(self):
+                self.x = 1
+                return 1
+            def m(self):
+                raise ValueError
+        class B(A):
+            def methodname(self):
+                return 2
+            def m(self):
+                return 3
+        def fancyname(a):
+            return a.methodname()
+        def m(a):
+            return a.m()
+        def h(flag):
+            if flag:
+                obj = A()
+            else:
+                obj = B()
+            fancyname(obj)
+            m(obj)
+
+        t, wa = self.translate(h, [int])
+        hgraph = graphof(t, h)
+        # fiiiish :-(
+        block = hgraph.startblock.exits[0].target.exits[0].target
+        op_call_fancyname = block.operations[0]
+
+        explanation = wa.explain_analyze_slowly(op_call_fancyname)
+        assert "fancyname" in explanation[0]
+        assert "methodname" in explanation[1]
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -151,8 +151,9 @@
                 # Also, ERROR confuses lib-python/conftest.py.
             raise CompilationError(stdout, stderr)
         else:
-            for line in stderr.splitlines():
-                log.WARNING(line)
+            if self.log_errors:
+                for line in stderr.splitlines():
+                    log.WARNING(line)
 
     def _make_o_file(self, cfile, ext):
         """Create an object file name under the udir for a .c file"""

From pypy.commits at gmail.com  Mon Sep 11 13:04:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 11 Sep 2017 10:04:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Speed up a slow test
Message-ID: <59b6c21a.37b9df0a.bf67e.2116@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92373:97f210be408f
Date: 2017-09-11 18:04 +0100
http://bitbucket.org/pypy/pypy/changeset/97f210be408f/

Log:	Speed up a slow test

diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -1092,8 +1092,6 @@
     def test_accumulate(self):
         """copied from ./lib-python/3/test/test_itertools.py"""
         from itertools import accumulate
-        from decimal import Decimal
-        from fractions import Fraction
         import _operator as operator
         expected = [0, 1, 3, 6, 10, 15, 21, 28, 36, 45]
         # one positional arg
@@ -1101,7 +1099,7 @@
         # kw arg
         assert list(accumulate(iterable=range(10))) == expected
         # multiple types
-        for typ in int, complex, Decimal, Fraction:
+        for typ in int, complex:
             assert list(accumulate(map(typ, range(10)))) == list(map(typ, expected))
         assert list(accumulate('abc')) == ['a', 'ab', 'abc']   # works with non-numeric
         assert list(accumulate([])) == []                  # empty iterable

From pypy.commits at gmail.com  Mon Sep 11 13:21:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 11 Sep 2017 10:21:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add space.long and space.w_long aliases
 and reduce diff with default
Message-ID: <59b6c631.138a1c0a.42db0.e2c0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92374:d9fa892ead1e
Date: 2017-09-11 18:21 +0100
http://bitbucket.org/pypy/pypy/changeset/d9fa892ead1e/

Log:	Add space.long and space.w_long aliases and reduce diff with default

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1625,6 +1625,7 @@
     def int(self, w_obj):
         assert w_obj is not None
         return w_obj.int(self)
+    long = int
 
     @specialize.argtype(1)
     def uint_w(self, w_obj):
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -6,8 +6,7 @@
 from pypy.interpreter.error import OperationError
 from rpython.rlib.rbigint import rbigint
 
-
-PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long", "w_int")
+PyLong_Check, PyLong_CheckExact = build_type_checkers_flags("Long")
 
 @cpython_api([lltype.Signed], PyObject)
 def PyLong_FromLong(space, val):
@@ -167,7 +166,7 @@
 @cpython_api([lltype.Float], PyObject)
 def PyLong_FromDouble(space, val):
     """Return a new PyLongObject object from v, or NULL on failure."""
-    return space.int(space.newfloat(val))
+    return space.long(space.newfloat(val))
 
 @cpython_api([PyObject], lltype.Float, error=-1.0)
 def PyLong_AsDouble(space, w_long):
@@ -192,7 +191,7 @@
     w_base = space.newint(rffi.cast(lltype.Signed, base))
     if pend:
         pend[0] = rffi.ptradd(str, len(s))
-    return space.call_function(space.w_int, w_str, w_base)
+    return space.call_function(space.w_long, w_str, w_base)
 
 @cpython_api([rffi.CWCHARP, Py_ssize_t, rffi.INT_real], PyObject)
 def PyLong_FromUnicode(space, u, length, base):
@@ -203,7 +202,7 @@
     out of range, ValueError will be raised."""
     w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
     w_base = space.newint(rffi.cast(lltype.Signed, base))
-    return space.call_function(space.w_int, w_value, w_base)
+    return space.call_function(space.w_long, w_value, w_base)
 
 @cpython_api([rffi.VOIDP], PyObject)
 def PyLong_FromVoidPtr(space, p):
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -95,6 +95,7 @@
         self.w_unicode = self.w_str
         self.w_text = self.w_str
         del self.w_str
+        self.w_long = self.w_int
         self.w_dict.flag_map_or_seq = 'M'
         from pypy.objspace.std import dictproxyobject
         dictproxyobject._set_flag_map_or_seq(self)

From pypy.commits at gmail.com  Mon Sep 11 15:40:40 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 12:40:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: add missing file
Message-ID: <59b6e6b8.cf97df0a.036c.034e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92375:303d4b69c445
Date: 2017-09-11 22:39 +0300
http://bitbucket.org/pypy/pypy/changeset/303d4b69c445/

Log:	add missing file

diff --git a/pypy/module/cpyext/src/unicodeobject.c b/pypy/module/cpyext/src/unicodeobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/unicodeobject.c
@@ -0,0 +1,423 @@
+
+#include "Python.h"
+static void
+makefmt(char *fmt, int longflag, int size_tflag, int zeropad, int width, int precision, char c)
+{
+    *fmt++ = '%';
+    if (width) {
+        if (zeropad)
+            *fmt++ = '0';
+        fmt += sprintf(fmt, "%d", width);
+    }
+    if (precision)
+        fmt += sprintf(fmt, ".%d", precision);
+    if (longflag)
+        *fmt++ = 'l';
+    else if (size_tflag) {
+        char *f = PY_FORMAT_SIZE_T;
+        while (*f)
+            *fmt++ = *f++;
+    }
+    *fmt++ = c;
+    *fmt = '\0';
+}
+
+#define appendstring(string) \
+    do { \
+        for (copy = string;*copy; copy++) { \
+            *s++ = (unsigned char)*copy; \
+        } \
+    } while (0)
+
+
+PyObject *
+PyUnicode_FromFormatV(const char *format, va_list vargs)
+{
+    va_list count;
+    Py_ssize_t callcount = 0;
+    PyObject **callresults = NULL;
+    PyObject **callresult = NULL;
+    Py_ssize_t n = 0;
+    int width = 0;
+    int precision = 0;
+    int zeropad;
+    const char* f;
+    Py_UNICODE *s;
+    PyObject *string;
+    /* used by sprintf */
+    char buffer[21];
+    /* use abuffer instead of buffer, if we need more space
+     * (which can happen if there's a format specifier with width). */
+    char *abuffer = NULL;
+    char *realbuffer;
+    Py_ssize_t abuffersize = 0;
+    char fmt[60]; /* should be enough for %0width.precisionld */
+    const char *copy;
+
+#ifdef VA_LIST_IS_ARRAY
+    Py_MEMCPY(count, vargs, sizeof(va_list));
+#else
+#ifdef  __va_copy
+    __va_copy(count, vargs);
+#else
+    count = vargs;
+#endif
+#endif
+     /* step 1: count the number of %S/%R/%s format specifications
+      * (we call PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() for these
+      * objects once during step 3 and put the result in an array) */
+    for (f = format; *f; f++) {
+         if (*f == '%') {
+             f++;
+             while (*f && *f != '%' && !isalpha((unsigned)*f))
+                 f++;
+             if (!*f)
+                 break;
+             if (*f == 's' || *f=='S' || *f=='R')
+                 ++callcount;
+         }
+    }
+    /* step 2: allocate memory for the results of
+     * PyObject_Str()/PyObject_Repr()/PyUnicode_DecodeUTF8() calls */
+    if (callcount) {
+        callresults = PyObject_Malloc(sizeof(PyObject *)*callcount);
+        if (!callresults) {
+            PyErr_NoMemory();
+            return NULL;
+        }
+        callresult = callresults;
+    }
+    /* step 3: figure out how large a buffer we need */
+    for (f = format; *f; f++) {
+        if (*f == '%') {
+            const char* p = f++;
+            width = 0;
+            while (isdigit((unsigned)*f))
+                width = (width*10) + *f++ - '0';
+            precision = 0;
+            if (*f == '.') {
+                f++;
+                while (isdigit((unsigned)*f))
+                    precision = (precision*10) + *f++ - '0';
+            }
+
+            /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since
+             * they don't affect the amount of space we reserve.
+             */
+            if ((*f == 'l' || *f == 'z') &&
+                (f[1] == 'd' || f[1] == 'u'))
+                ++f;
+
+            switch (*f) {
+            case 'c':
+            {
+                int ordinal = va_arg(count, int);
+#ifdef Py_UNICODE_WIDE
+                if (ordinal < 0 || ordinal > 0x10ffff) {
+                    PyErr_SetString(PyExc_OverflowError,
+                                    "%c arg not in range(0x110000) "
+                                    "(wide Python build)");
+                    goto fail;
+                }
+#else
+                if (ordinal < 0 || ordinal > 0xffff) {
+                    PyErr_SetString(PyExc_OverflowError,
+                                    "%c arg not in range(0x10000) "
+                                    "(narrow Python build)");
+                    goto fail;
+                }
+#endif
+                /* fall through... */
+            }
+            case '%':
+                n++;
+                break;
+            case 'd': case 'u': case 'i': case 'x':
+                (void) va_arg(count, int);
+                if (width < precision)
+                    width = precision;
+                /* 20 bytes is enough to hold a 64-bit
+                   integer.  Decimal takes the most space.
+                   This isn't enough for octal.
+                   If a width is specified we need more
+                   (which we allocate later). */
+                if (width < 20)
+                    width = 20;
+                n += width;
+                if (abuffersize < width)
+                    abuffersize = width;
+                break;
+            case 's':
+            {
+                /* UTF-8 */
+                const char *s = va_arg(count, const char*);
+                PyObject *str = PyUnicode_DecodeUTF8(s, strlen(s), "replace");
+                if (!str)
+                    goto fail;
+                n += PyUnicode_GET_SIZE(str);
+                /* Remember the str and switch to the next slot */
+                *callresult++ = str;
+                break;
+            }
+            case 'U':
+            {
+                PyObject *obj = va_arg(count, PyObject *);
+                assert(obj && PyUnicode_Check(obj));
+                n += PyUnicode_GET_SIZE(obj);
+                break;
+            }
+            case 'V':
+            {
+                PyObject *obj = va_arg(count, PyObject *);
+                const char *str = va_arg(count, const char *);
+                assert(obj || str);
+                assert(!obj || PyUnicode_Check(obj));
+                if (obj)
+                    n += PyUnicode_GET_SIZE(obj);
+                else
+                    n += strlen(str);
+                break;
+            }
+            case 'S':
+            {
+                PyObject *obj = va_arg(count, PyObject *);
+                PyObject *str;
+                assert(obj);
+                str = PyObject_Str(obj);
+                if (!str)
+                    goto fail;
+                n += PyString_GET_SIZE(str);
+                /* Remember the str and switch to the next slot */
+                *callresult++ = str;
+                break;
+            }
+            case 'R':
+            {
+                PyObject *obj = va_arg(count, PyObject *);
+                PyObject *repr;
+                assert(obj);
+                repr = PyObject_Repr(obj);
+                if (!repr)
+                    goto fail;
+                n += PyUnicode_GET_SIZE(repr);
+                /* Remember the repr and switch to the next slot */
+                *callresult++ = repr;
+                break;
+            }
+            case 'p':
+                (void) va_arg(count, int);
+                /* maximum 64-bit pointer representation:
+                 * 0xffffffffffffffff
+                 * so 19 characters is enough.
+                 * XXX I count 18 -- what's the extra for?
+                 */
+                n += 19;
+                break;
+            default:
+                /* if we stumble upon an unknown
+                   formatting code, copy the rest of
+                   the format string to the output
+                   string. (we cannot just skip the
+                   code, since there's no way to know
+                   what's in the argument list) */
+                n += strlen(p);
+                goto expand;
+            }
+        } else
+            n++;
+    }
+  expand:
+    if (abuffersize > 20) {
+        /* add 1 for sprintf's trailing null byte */
+        abuffer = PyObject_Malloc(abuffersize + 1);
+        if (!abuffer) {
+            PyErr_NoMemory();
+            goto fail;
+        }
+        realbuffer = abuffer;
+    }
+    else
+        realbuffer = buffer;
+    /* step 4: fill the buffer */
+    /* Since we've analyzed how much space we need for the worst case,
+       we don't have to resize the string.
+       There can be no errors beyond this point. */
+    string = PyUnicode_FromUnicode(NULL, n);
+    if (!string)
+        goto fail;
+
+    s = PyUnicode_AS_UNICODE(string);
+    callresult = callresults;
+
+    for (f = format; *f; f++) {
+        if (*f == '%') {
+            const char* p = f++;
+            int longflag = 0;
+            int size_tflag = 0;
+            zeropad = (*f == '0');
+            /* parse the width.precision part */
+            width = 0;
+            while (isdigit((unsigned)*f))
+                width = (width*10) + *f++ - '0';
+            precision = 0;
+            if (*f == '.') {
+                f++;
+                while (isdigit((unsigned)*f))
+                    precision = (precision*10) + *f++ - '0';
+            }
+            /* handle the long flag, but only for %ld and %lu.
+               others can be added when necessary. */
+            if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) {
+                longflag = 1;
+                ++f;
+            }
+            /* handle the size_t flag. */
+            if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) {
+                size_tflag = 1;
+                ++f;
+            }
+
+            switch (*f) {
+            case 'c':
+                *s++ = va_arg(vargs, int);
+                break;
+            case 'd':
+                makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'd');
+                if (longflag)
+                    sprintf(realbuffer, fmt, va_arg(vargs, long));
+                else if (size_tflag)
+                    sprintf(realbuffer, fmt, va_arg(vargs, Py_ssize_t));
+                else
+                    sprintf(realbuffer, fmt, va_arg(vargs, int));
+                appendstring(realbuffer);
+                break;
+            case 'u':
+                makefmt(fmt, longflag, size_tflag, zeropad, width, precision, 'u');
+                if (longflag)
+                    sprintf(realbuffer, fmt, va_arg(vargs, unsigned long));
+                else if (size_tflag)
+                    sprintf(realbuffer, fmt, va_arg(vargs, size_t));
+                else
+                    sprintf(realbuffer, fmt, va_arg(vargs, unsigned int));
+                appendstring(realbuffer);
+                break;
+            case 'i':
+                makefmt(fmt, 0, 0, zeropad, width, precision, 'i');
+                sprintf(realbuffer, fmt, va_arg(vargs, int));
+                appendstring(realbuffer);
+                break;
+            case 'x':
+                makefmt(fmt, 0, 0, zeropad, width, precision, 'x');
+                sprintf(realbuffer, fmt, va_arg(vargs, int));
+                appendstring(realbuffer);
+                break;
+            case 's':
+            {
+                /* unused, since we already have the result */
+                (void) va_arg(vargs, char *);
+                Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(*callresult),
+                                PyUnicode_GET_SIZE(*callresult));
+                s += PyUnicode_GET_SIZE(*callresult);
+                /* We're done with the unicode()/repr() => forget it */
+                Py_DECREF(*callresult);
+                /* switch to next unicode()/repr() result */
+                ++callresult;
+                break;
+            }
+            case 'U':
+            {
+                PyObject *obj = va_arg(vargs, PyObject *);
+                Py_ssize_t size = PyUnicode_GET_SIZE(obj);
+                Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size);
+                s += size;
+                break;
+            }
+            case 'V':
+            {
+                PyObject *obj = va_arg(vargs, PyObject *);
+                const char *str = va_arg(vargs, const char *);
+                if (obj) {
+                    Py_ssize_t size = PyUnicode_GET_SIZE(obj);
+                    Py_UNICODE_COPY(s, PyUnicode_AS_UNICODE(obj), size);
+                    s += size;
+                } else {
+                    appendstring(str);
+                }
+                break;
+            }
+            case 'S':
+            case 'R':
+            {
+                const char *str = PyString_AS_STRING(*callresult);
+                /* unused, since we already have the result */
+                (void) va_arg(vargs, PyObject *);
+                appendstring(str);
+                /* We're done with the unicode()/repr() => forget it */
+                Py_DECREF(*callresult);
+                /* switch to next unicode()/repr() result */
+                ++callresult;
+                break;
+            }
+            case 'p':
+                sprintf(buffer, "%p", va_arg(vargs, void*));
+                /* %p is ill-defined:  ensure leading 0x. */
+                if (buffer[1] == 'X')
+                    buffer[1] = 'x';
+                else if (buffer[1] != 'x') {
+                    memmove(buffer+2, buffer, strlen(buffer)+1);
+                    buffer[0] = '0';
+                    buffer[1] = 'x';
+                }
+                appendstring(buffer);
+                break;
+            case '%':
+                *s++ = '%';
+                break;
+            default:
+                appendstring(p);
+                goto end;
+            }
+        } else
+            *s++ = *f;
+    }
+
+  end:
+    if (callresults)
+        PyObject_Free(callresults);
+    if (abuffer)
+        PyObject_Free(abuffer);
+    PyUnicode_Resize(&string, s - PyUnicode_AS_UNICODE(string));
+    return string;
+  fail:
+    if (callresults) {
+        PyObject **callresult2 = callresults;
+        while (callresult2 < callresult) {
+            Py_DECREF(*callresult2);
+            ++callresult2;
+        }
+        PyObject_Free(callresults);
+    }
+    if (abuffer)
+        PyObject_Free(abuffer);
+    return NULL;
+}
+
+#undef appendstring
+
+PyObject *
+PyUnicode_FromFormat(const char *format, ...)
+{
+    PyObject* ret;
+    va_list vargs;
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    ret = PyUnicode_FromFormatV(format, vargs);
+    va_end(vargs);
+    return ret;
+}
+
+

From pypy.commits at gmail.com  Mon Sep 11 16:51:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 11 Sep 2017 13:51:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix,
 on 32 bit platforms 1L<<31 overflows a 'l' parsed object
Message-ID: <59b6f751.c35c1c0a.b4737.dcfd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92376:eeb144e01b46
Date: 2017-09-11 23:48 +0300
http://bitbucket.org/pypy/pypy/changeset/eeb144e01b46/

Log:	fix, on 32 bit platforms 1L<<31 overflows a 'l' parsed object

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1384,9 +1384,9 @@
         module = self.import_extension('foo', [
            ("test_flags", "METH_VARARGS",
             '''
-                long in_flag, my_flag;
+                long long in_flag, my_flag;
                 PyObject * obj;
-                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag))
                     return NULL;
                 if (!PyType_Check(obj))
                 {

From pypy.commits at gmail.com  Wed Sep 13 02:38:15 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 12 Sep 2017 23:38:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: update build instructions for pyexpat
Message-ID: <59b8d257.020a1c0a.b1326.0f7c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92377:8f23ddc1eec0
Date: 2017-09-13 09:36 +0300
http://bitbucket.org/pypy/pypy/changeset/8f23ddc1eec0/

Log:	update build instructions for pyexpat

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -209,17 +209,85 @@
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
-Download the source code of expat on sourceforge:
-https://github.com/libexpat/libexpat/releases and extract it in the base directory.
-Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
-with Visual Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for
-the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes)
+CPython compiles expat from source as part of the build. PyPy uses the same
+code base, but expects to link to a static lib of expat. Here are instructions
+to reproduce the static lib in version 2.2.4.
 
-Then, copy the file ``win32\bin\release\libexpat.lib`` into
-LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in
-INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH.
+Download the source code of expat: https://github.com/libexpat/libexpat. 
+``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
+``vcvars.bat`` to set up the visual compiler tools, and CD into the source
+directory. Create a file ``stdbool.h`` with the content
+
+.. code-block:: c
+
+    #pragma once
+
+    #define false   0
+    #define true    1
+
+    #define bool int
+
+and put it in a place on the ``INCLUDE`` path, or create it in the local
+directory and add ``.`` to the ``INCLUDE`` path::
+
+    SET INCLUDE=%INCLUDE%;.
+
+Then compile all the ``*.c`` file into ``*.obj``::
+
+    cl.exe /nologo /MD  /O2 *c /c
+    rem for debug
+    cl.exe /nologo /MD  /O0 /Ob0 /Zi *c /c
+
+You may need to move some variable declarations to the beginning of the
+function, to be compliant with C89 standard. Here is the diff for version 2.2.4
+
+.. code-block:: diff
+
+    diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
+    index 007aed0..a2dcaad 100644
+    --- a/expat/lib/xmltok.c
+    +++ b/expat/lib/xmltok.c
+    @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
+       /* Avoid copying partial characters (due to limited space). */
+       const ptrdiff_t bytesAvailable = fromLim - *fromP;
+       const ptrdiff_t bytesStorable = toLim - *toP;
+    +  const char * fromLimBefore;
+    +  ptrdiff_t bytesToCopy;
+       if (bytesAvailable > bytesStorable) {
+         fromLim = *fromP + bytesStorable;
+         output_exhausted = true;
+       }
+
+       /* Avoid copying partial characters (from incomplete input). */
+    -  const char * const fromLimBefore = fromLim;
+    +  fromLimBefore = fromLim;
+       align_limit_to_full_utf8_characters(*fromP, &fromLim);
+       if (fromLim < fromLimBefore) {
+         input_incomplete = true;
+       }
+
+    -  const ptrdiff_t bytesToCopy = fromLim - *fromP;
+    +  bytesToCopy = fromLim - *fromP;
+       memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
+       *fromP += bytesToCopy;
+       *toP += bytesToCopy;
+
+
+Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
+
+    cl /LD *.obj libexpat.def /Felibexpat.dll 
+    rem for debug
+    rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
+
+    rem this will override the export library created in the step above
+    rem but tests do not need the export library, they load the dll dynamically
+    lib *.obj /out:libexpat.lib
+
+Then, copy 
+
+- ``libexpat.lib`` into LIB
+- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
+- ``libexpat.dll`` into PATH
 
 
 The OpenSSL library
@@ -363,7 +431,7 @@
 It is probably not too much work if the goal is only to get a translated
 PyPy executable, and to run all tests before translation.  But you need
 to start somewhere, and you should start with some tests in
-rpython/translator/c/test/, like ``test_standalone.py`` and
+``rpython/translator/c/test/``, like ``test_standalone.py`` and
 ``test_newgc.py``: try to have them pass on top of CPython64/64.
 
 Keep in mind that this runs small translations, and some details may go
@@ -373,7 +441,7 @@
 should be something like ``long long``.
 
 What is more generally needed is to review all the C files in
-rpython/translator/c/src for the word ``long``, because this means a
+``rpython/translator/c/src`` for the word ``long``, because this means a
 32-bit integer even on Win64.  Replace it with ``Signed`` most of the
 times.  You can replace one with the other without breaking anything on
 any other platform, so feel free to.

From pypy.commits at gmail.com  Wed Sep 13 12:48:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Sep 2017 09:48:22 -0700 (PDT)
Subject: [pypy-commit] buildbot cleanup-hg-bookmarks: Tentative Windows
 version 2
Message-ID: <59b96156.6588df0a.6a7a0.7051@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cleanup-hg-bookmarks
Changeset: r1025:ad784c41e04a
Date: 2017-09-13 19:48 +0300
http://bitbucket.org/pypy/buildbot/changeset/ad784c41e04a/

Log:	Tentative Windows version 2

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -347,7 +347,7 @@
         factory.addStep(ShellCmd(
             description="cleanup bookmarks",
             command=["rm", "-f", ".hg/bookmarks"] if platform != 'win32'
-               else ["if exist .hg/bookmarks del .hg/bookmarks"],
+               else [r"cmd /c if exist .hg\bookmarks del .hg\bookmarks"],
             workdir=workdir,
             haltOnFailure=False,
         ))

From pypy.commits at gmail.com  Wed Sep 13 16:41:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 13 Sep 2017 13:41:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Tweak inspector.get_rpy_roots()
Message-ID: <59b99813.6594df0a.b3998.3e7e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92378:f3070ec0eeb3
Date: 2017-09-13 22:36 +0200
http://bitbucket.org/pypy/pypy/changeset/f3070ec0eeb3/

Log:	Tweak inspector.get_rpy_roots()

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,6 +22,7 @@
     can_usually_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -10,73 +10,62 @@
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_roots() ----------
 
-def _counting_rpy_root(obj, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_roots(gc):
-    gc._count_rpy = 0
-    gc.enumerate_all_roots(_counting_rpy_root, gc)
-    return gc._count_rpy
-
 def _append_rpy_root(obj, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected at the end
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
 def _do_append_rpy_roots(gc, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.enumerate_all_roots(_append_rpy_root, gc)
     gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_roots(gc):
-    count = _do_count_rpy_roots(gc)
-    extra = 16
     while True:
-        result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
-        try:
-            _do_append_rpy_roots(gc, result)
-        except ValueError:
-            extra *= 3
-        else:
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * gc._totalroots_rpy
+        count = _do_append_rpy_roots(gc, result)
+        if count <= len(result):     # 'count' fits inside the list
             return result
+        count += (count // 8)
+        gc._totalroots_rpy = count + 10
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_referents() ----------
 
-def _count_rpy_referent(pointer, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_referents(gc, gcref):
-    gc._count_rpy = 0
-    gc.trace(llmemory.cast_ptr_to_adr(gcref), _count_rpy_referent, gc)
-    return gc._count_rpy
-
 def _append_rpy_referent(pointer, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
+                                              llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected at the end
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
-                                          llmemory.GCREF)
 
 def _do_append_rpy_referents(gc, gcref, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.trace(llmemory.cast_ptr_to_adr(gcref), _append_rpy_referent, gc)
+    gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_referents(gc, gcref):
-    count = _do_count_rpy_referents(gc, gcref)
-    result = [lltype.nullptr(llmemory.GCREF.TO)] * count
-    _do_append_rpy_referents(gc, gcref, result)
-    return result
+    count = 7
+    while True:
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * count
+        count = _do_append_rpy_referents(gc, gcref, result)
+        if count <= len(result):     # 'count' fits inside the list
+            return result
+        count += (count // 8)
 
 # ----------
 

From pypy.commits at gmail.com  Wed Sep 13 16:41:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 13 Sep 2017 13:41:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <59b99815.84e51c0a.17488.535e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92379:d8cf1127fabb
Date: 2017-09-13 22:41 +0200
http://bitbucket.org/pypy/pypy/changeset/d8cf1127fabb/

Log:	merge heads

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -209,17 +209,85 @@
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
-Download the source code of expat on sourceforge:
-https://github.com/libexpat/libexpat/releases and extract it in the base directory.
-Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
-with Visual Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for
-the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes)
+CPython compiles expat from source as part of the build. PyPy uses the same
+code base, but expects to link to a static lib of expat. Here are instructions
+to reproduce the static lib in version 2.2.4.
 
-Then, copy the file ``win32\bin\release\libexpat.lib`` into
-LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in
-INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH.
+Download the source code of expat: https://github.com/libexpat/libexpat. 
+``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
+``vcvars.bat`` to set up the visual compiler tools, and CD into the source
+directory. Create a file ``stdbool.h`` with the content
+
+.. code-block:: c
+
+    #pragma once
+
+    #define false   0
+    #define true    1
+
+    #define bool int
+
+and put it in a place on the ``INCLUDE`` path, or create it in the local
+directory and add ``.`` to the ``INCLUDE`` path::
+
+    SET INCLUDE=%INCLUDE%;.
+
+Then compile all the ``*.c`` file into ``*.obj``::
+
+    cl.exe /nologo /MD  /O2 *c /c
+    rem for debug
+    cl.exe /nologo /MD  /O0 /Ob0 /Zi *c /c
+
+You may need to move some variable declarations to the beginning of the
+function, to be compliant with C89 standard. Here is the diff for version 2.2.4
+
+.. code-block:: diff
+
+    diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
+    index 007aed0..a2dcaad 100644
+    --- a/expat/lib/xmltok.c
+    +++ b/expat/lib/xmltok.c
+    @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
+       /* Avoid copying partial characters (due to limited space). */
+       const ptrdiff_t bytesAvailable = fromLim - *fromP;
+       const ptrdiff_t bytesStorable = toLim - *toP;
+    +  const char * fromLimBefore;
+    +  ptrdiff_t bytesToCopy;
+       if (bytesAvailable > bytesStorable) {
+         fromLim = *fromP + bytesStorable;
+         output_exhausted = true;
+       }
+
+       /* Avoid copying partial characters (from incomplete input). */
+    -  const char * const fromLimBefore = fromLim;
+    +  fromLimBefore = fromLim;
+       align_limit_to_full_utf8_characters(*fromP, &fromLim);
+       if (fromLim < fromLimBefore) {
+         input_incomplete = true;
+       }
+
+    -  const ptrdiff_t bytesToCopy = fromLim - *fromP;
+    +  bytesToCopy = fromLim - *fromP;
+       memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
+       *fromP += bytesToCopy;
+       *toP += bytesToCopy;
+
+
+Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
+
+    cl /LD *.obj libexpat.def /Felibexpat.dll 
+    rem for debug
+    rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
+
+    rem this will override the export library created in the step above
+    rem but tests do not need the export library, they load the dll dynamically
+    lib *.obj /out:libexpat.lib
+
+Then, copy 
+
+- ``libexpat.lib`` into LIB
+- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
+- ``libexpat.dll`` into PATH
 
 
 The OpenSSL library
@@ -363,7 +431,7 @@
 It is probably not too much work if the goal is only to get a translated
 PyPy executable, and to run all tests before translation.  But you need
 to start somewhere, and you should start with some tests in
-rpython/translator/c/test/, like ``test_standalone.py`` and
+``rpython/translator/c/test/``, like ``test_standalone.py`` and
 ``test_newgc.py``: try to have them pass on top of CPython64/64.
 
 Keep in mind that this runs small translations, and some details may go
@@ -373,7 +441,7 @@
 should be something like ``long long``.
 
 What is more generally needed is to review all the C files in
-rpython/translator/c/src for the word ``long``, because this means a
+``rpython/translator/c/src`` for the word ``long``, because this means a
 32-bit integer even on Win64.  Replace it with ``Signed`` most of the
 times.  You can replace one with the other without breaking anything on
 any other platform, so feel free to.

From pypy.commits at gmail.com  Wed Sep 13 19:46:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 13 Sep 2017 16:46:15 -0700 (PDT)
Subject: [pypy-commit] pypy default: Found out that gc.get_objects() returns
 some, but not all, 
Message-ID: <59b9c347.9598df0a.19853.b6d5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92380:90a0d3659179
Date: 2017-09-14 00:32 +0200
http://bitbucket.org/pypy/pypy/changeset/90a0d3659179/

Log:	Found out that gc.get_objects() returns some, but not all, objects
	with finalizers, in some strange way. More precisely, it completely
	misses objects with finalizers that have recently become
	unreachable, but it no longer misses these objects after the next
	major collection occurred.

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -336,6 +336,7 @@
         callback2, attrname = _convert_callback_formats(callback)    # :-/
         setattr(self, attrname, arg)
         self.root_walker.walk_roots(callback2, callback2, callback2)
+        self.enum_live_with_finalizers(callback, arg)
         self.enum_pending_finalizers(callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
@@ -348,6 +349,12 @@
             i += 1
     enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        # as far as possible, enumerates the live objects with finalizers,
+        # even if they have not been detected as unreachable yet (but may be)
+        pass
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _copy_pending_finalizers_deque(self, deque, copy_fn):
         tmp = self.AddressDeque()
         while deque.non_empty():
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2515,6 +2515,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _collect_obj(self, obj, ignored):
         # Ignore pinned objects, which are the ones still in the nursery here.
         # Cache effects: don't read any flag out of 'obj' at this point.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1780,6 +1780,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     @staticmethod
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -269,22 +269,23 @@
             self.index_in_oldest = index + 1
             return result
 
-        def foreach(self, callback, arg):
+        def foreach(self, callback, arg, step=1):
             """Invoke 'callback(address, arg)' for all addresses in the deque.
             Typically, 'callback' is a bound method and 'arg' can be None.
+            If step > 1, only calls it for addresses multiple of 'step'.
             """
             chunk = self.oldest_chunk
             index = self.index_in_oldest
             while chunk is not self.newest_chunk:
                 while index < chunk_size:
                     callback(chunk.items[index], arg)
-                    index += 1
+                    index += step
                 chunk = chunk.next
-                index = 0
+                index -= chunk_size
             limit = self.index_in_newest
             while index < limit:
                 callback(chunk.items[index], arg)
-                index += 1
+                index += step
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def delete(self):
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -160,6 +160,10 @@
 
             ll.foreach(callback, 42)
             assert seen == addrs
+            seen = []
+            ll.foreach(callback, 42, step=2)
+            assert seen == addrs[::2]
+
             for a in addrs:
                 b = ll.popleft()
                 assert a == b

From pypy.commits at gmail.com  Wed Sep 13 19:46:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 13 Sep 2017 16:46:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Temporarily backout 90a0d3659179 and
 f3070ec0eeb3, which cause failures
Message-ID: <59b9c349.865b1c0a.e58f2.04cd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92381:626793f5423f
Date: 2017-09-14 01:45 +0200
http://bitbucket.org/pypy/pypy/changeset/626793f5423f/

Log:	Temporarily backout 90a0d3659179 and f3070ec0eeb3, which cause
	failures

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,7 +22,6 @@
     can_usually_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
-    _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
@@ -336,7 +335,6 @@
         callback2, attrname = _convert_callback_formats(callback)    # :-/
         setattr(self, attrname, arg)
         self.root_walker.walk_roots(callback2, callback2, callback2)
-        self.enum_live_with_finalizers(callback, arg)
         self.enum_pending_finalizers(callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
@@ -349,12 +347,6 @@
             i += 1
     enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
-    def enum_live_with_finalizers(self, callback, arg):
-        # as far as possible, enumerates the live objects with finalizers,
-        # even if they have not been detected as unreachable yet (but may be)
-        pass
-    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
-
     def _copy_pending_finalizers_deque(self, deque, copy_fn):
         tmp = self.AddressDeque()
         while deque.non_empty():
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2515,11 +2515,6 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
-    def enum_live_with_finalizers(self, callback, arg):
-        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
-        self.old_objects_with_finalizers.foreach(callback, arg, 2)
-    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
-
     def _collect_obj(self, obj, ignored):
         # Ignore pinned objects, which are the ones still in the nursery here.
         # Cache effects: don't read any flag out of 'obj' at this point.
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -10,62 +10,73 @@
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_roots() ----------
 
+def _counting_rpy_root(obj, gc):
+    gc._count_rpy += 1
+
+def _do_count_rpy_roots(gc):
+    gc._count_rpy = 0
+    gc.enumerate_all_roots(_counting_rpy_root, gc)
+    return gc._count_rpy
+
 def _append_rpy_root(obj, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index < len(lst):
-        lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
-    #else:
-    #   too many items.  This situation is detected at the end
+    if index >= len(lst):
+        raise ValueError
     gc._count_rpy = index + 1
+    lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
 
 def _do_append_rpy_roots(gc, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.enumerate_all_roots(_append_rpy_root, gc)
     gc._list_rpy = None
-    return gc._count_rpy
 
 def get_rpy_roots(gc):
+    count = _do_count_rpy_roots(gc)
+    extra = 16
     while True:
-        result = [lltype.nullptr(llmemory.GCREF.TO)] * gc._totalroots_rpy
-        count = _do_append_rpy_roots(gc, result)
-        if count <= len(result):     # 'count' fits inside the list
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
+        try:
+            _do_append_rpy_roots(gc, result)
+        except ValueError:
+            extra *= 3
+        else:
             return result
-        count += (count // 8)
-        gc._totalroots_rpy = count + 10
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_referents() ----------
 
+def _count_rpy_referent(pointer, gc):
+    gc._count_rpy += 1
+
+def _do_count_rpy_referents(gc, gcref):
+    gc._count_rpy = 0
+    gc.trace(llmemory.cast_ptr_to_adr(gcref), _count_rpy_referent, gc)
+    return gc._count_rpy
+
 def _append_rpy_referent(pointer, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index < len(lst):
-        lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
-                                              llmemory.GCREF)
-    #else:
-    #   too many items.  This situation is detected at the end
+    if index >= len(lst):
+        raise ValueError
     gc._count_rpy = index + 1
+    lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
+                                          llmemory.GCREF)
 
 def _do_append_rpy_referents(gc, gcref, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.trace(llmemory.cast_ptr_to_adr(gcref), _append_rpy_referent, gc)
-    gc._list_rpy = None
-    return gc._count_rpy
 
 def get_rpy_referents(gc, gcref):
-    count = 7
-    while True:
-        result = [lltype.nullptr(llmemory.GCREF.TO)] * count
-        count = _do_append_rpy_referents(gc, gcref, result)
-        if count <= len(result):     # 'count' fits inside the list
-            return result
-        count += (count // 8)
+    count = _do_count_rpy_referents(gc, gcref)
+    result = [lltype.nullptr(llmemory.GCREF.TO)] * count
+    _do_append_rpy_referents(gc, gcref, result)
+    return result
 
 # ----------
 
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1780,11 +1780,6 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
-    def enum_live_with_finalizers(self, callback, arg):
-        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
-        self.old_objects_with_finalizers.foreach(callback, arg, 2)
-    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
-
     @staticmethod
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -269,23 +269,22 @@
             self.index_in_oldest = index + 1
             return result
 
-        def foreach(self, callback, arg, step=1):
+        def foreach(self, callback, arg):
             """Invoke 'callback(address, arg)' for all addresses in the deque.
             Typically, 'callback' is a bound method and 'arg' can be None.
-            If step > 1, only calls it for addresses multiple of 'step'.
             """
             chunk = self.oldest_chunk
             index = self.index_in_oldest
             while chunk is not self.newest_chunk:
                 while index < chunk_size:
                     callback(chunk.items[index], arg)
-                    index += step
+                    index += 1
                 chunk = chunk.next
-                index -= chunk_size
+                index = 0
             limit = self.index_in_newest
             while index < limit:
                 callback(chunk.items[index], arg)
-                index += step
+                index += 1
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def delete(self):
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -160,10 +160,6 @@
 
             ll.foreach(callback, 42)
             assert seen == addrs
-            seen = []
-            ll.foreach(callback, 42, step=2)
-            assert seen == addrs[::2]
-
             for a in addrs:
                 b = ll.popleft()
                 assert a == b

From pypy.commits at gmail.com  Thu Sep 14 00:41:51 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Sep 2017 21:41:51 -0700 (PDT)
Subject: [pypy-commit] pypy win32-fixes6: import earlier
Message-ID: <59ba088f.c55c1c0a.be23f.3b6f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-fixes6
Changeset: r92384:08a9cccce3f5
Date: 2017-09-14 06:30 +0300
http://bitbucket.org/pypy/pypy/changeset/08a9cccce3f5/

Log:	import earlier

diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -214,7 +214,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)

From pypy.commits at gmail.com  Thu Sep 14 00:41:53 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Sep 2017 21:41:53 -0700 (PDT)
Subject: [pypy-commit] pypy win32-fixes6: ensure disabled vmprof can skip
 tests
Message-ID: <59ba0891.c55d1c0a.29ef7.93df@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-fixes6
Changeset: r92385:049fa1d8e804
Date: 2017-09-14 06:46 +0300
http://bitbucket.org/pypy/pypy/changeset/049fa1d8e804/

Log:	ensure disabled vmprof can skip tests

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -3,6 +3,7 @@
 from pypy.interpreter.pyframe import PyFrame
 from pypy.interpreter.pycode import PyCode
 from pypy.interpreter.baseobjspace import W_Root
+from rpython.translator.platform import CompilationError
 from rpython.rlib import rvmprof, jit
 from pypy.interpreter.error import oefmt
 
@@ -11,8 +12,10 @@
 
 _get_code = lambda frame, w_inputvalue, operr: frame.pycode
 _decorator = rvmprof.vmprof_execute_code("pypy", _get_code, W_Root)
-my_execute_frame = _decorator(PyFrame.execute_frame)
-
+try:
+    my_execute_frame = _decorator(PyFrame.execute_frame)
+except CompilationError:
+    raise rvmprof.VMProfPlatformUnsupported('compilation failed')
 
 class __extend__(PyFrame):
     def execute_frame(self, w_inputvalue=None, operr=None):
diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -1,4 +1,7 @@
-import sys
+import sys, py
+if sys.platform == 'win32':
+    py.test.skip('vmprof disabled on windows')
+
 from rpython.tool.udir import udir
 from pypy.tool.pytest.objspace import gettestobjspace
 
diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py
--- a/pypy/module/_vmprof/test/test_direct.py
+++ b/pypy/module/_vmprof/test/test_direct.py
@@ -5,6 +5,9 @@
 except ImportError:
     py.test.skip('cffi required')
 
+if sys.platform == 'win32':
+    py.test.skip('vmprof disabled on windows')
+
 from rpython.rlib import rvmprof
 srcdir = py.path.local(rvmprof.__file__).join("..", "src")
 shareddir = srcdir.join('shared')

From pypy.commits at gmail.com  Thu Sep 14 00:41:47 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Sep 2017 21:41:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak windows builds and docs
Message-ID: <59ba088b.2395df0a.e7987.f64a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92382:17c82fc38ec9
Date: 2017-09-13 19:53 +0300
http://bitbucket.org/pypy/pypy/changeset/17c82fc38ec9/

Log:	tweak windows builds and docs

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -114,12 +114,15 @@
 INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
 
 
-Abridged method (for -Ojit builds using Visual Studio 2008)
------------------------------------------------------------
+Abridged method (using Visual Studio 2008)
+------------------------------------------
 
 Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_59.zip
+(for post-5.8 builds) with sha256 checksum
+``0f96c045db1f5f73ad0fae7857caa69c261324bd8e51f6d2ad1fa842c4a5f26f``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(for post-5.7.1 builds) with sha256 checksum 
+(to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
@@ -135,8 +138,8 @@
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
 
-Nonabrided method (building from scratch)
------------------------------------------
+Nonabridged method (building from scratch)
+------------------------------------------
 
 If you want to, you can rebuild everything from scratch by continuing.
 
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -488,19 +488,19 @@
                 deps.append('icon.res')
                 wdeps.append('icon.res')
             m.rule('$(DEFAULT_TARGET)', ['$(TARGET)'] + deps,
-                   ['$(CC_LINK) /nologo /debug %s ' % (' '.join(deps),) + \
+                   ['$(CC_LINK) /nologo /debug /LARGEADDRESSAWARE %s ' % (' '.join(deps),) + \
                     '$(SHARED_IMPORT_LIB) /out:$@ ' + \
                     '/MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
             m.rule('$(WTARGET)', ['$(TARGET)'] + wdeps,
-                   ['$(CC_LINK) /nologo /debug /SUBSYSTEM:WINDOWS %s ' % (' '.join(wdeps),) + \
-                    '$(SHARED_IMPORT_LIB) /out:$@ ' + \
+                   ['$(CC_LINK) /nologo /debug /LARGEADDRESSAWARE /SUBSYSTEM:WINDOWS %s ' % (
+                    ' '.join(wdeps),) + '$(SHARED_IMPORT_LIB) /out:$@ ' + \
                     '/MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
             m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)']+deps,
-                   ['$(CC_LINK) /nologo /DEBUG %s ' % (' '.join(deps),) + \
+                   ['$(CC_LINK) /nologo /DEBUG /LARGEADDRESSAWARE %s ' % (' '.join(deps),) + \
                     'debugmode_$(SHARED_IMPORT_LIB) /out:$@',
                     ])
 

From pypy.commits at gmail.com  Thu Sep 14 00:41:49 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 13 Sep 2017 21:41:49 -0700 (PDT)
Subject: [pypy-commit] pypy win32-fixes6: more win32 fixes
Message-ID: <59ba088d.cd5e1c0a.1a53f.802a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-fixes6
Changeset: r92383:b48803972c04
Date: 2017-09-14 07:40 +0300
http://bitbucket.org/pypy/pypy/changeset/b48803972c04/

Log:	more win32 fixes


From pypy.commits at gmail.com  Thu Sep 14 04:53:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 14 Sep 2017 01:53:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Test and fix: an ll_assert() can rarely
 fail in debug mode in the GC, 
Message-ID: <59ba4383.4ad61c0a.2aa1a.32e1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92386:9bd6fef43680
Date: 2017-09-14 10:52 +0200
http://bitbucket.org/pypy/pypy/changeset/9bd6fef43680/

Log:	Test and fix: an ll_assert() can rarely fail in debug mode in the
	GC, but that's not a real problem

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1897,8 +1897,14 @@
                         if cardbyte & 1:
                             if interval_stop > length:
                                 interval_stop = length
-                                ll_assert(cardbyte <= 1 and bytes == 0,
-                                          "premature end of object")
+                                #--- the sanity check below almost always
+                                #--- passes, except in situations like
+                                #--- test_writebarrier_before_copy_manually\
+                                #    _copy_card_bits
+                                #ll_assert(cardbyte <= 1 and bytes == 0,
+                                #          "premature end of object")
+                                ll_assert(bytes == 0, "premature end of object")
+                                cardbyte = 1
                             self.trace_and_drag_out_of_nursery_partial(
                                 obj, interval_start, interval_stop)
                         #
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1399,8 +1399,15 @@
                         if cardbyte & 1:
                             if interval_stop > length:
                                 interval_stop = length
-                                ll_assert(cardbyte <= 1 and bytes == 0,
-                                          "premature end of object")
+                                #--- the sanity check below almost always
+                                #--- passes, except in situations like
+                                #--- test_writebarrier_before_copy_manually\
+                                #    _copy_card_bits
+                                #ll_assert(cardbyte <= 1 and bytes == 0,
+                                #          "premature end of object")
+                                ll_assert(bytes == 0, "premature end of object")
+                                if interval_stop <= interval_start:
+                                    break
                             self.trace_and_drag_out_of_nursery_partial(
                                 obj, interval_start, interval_stop)
                         #
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -995,6 +995,31 @@
 
         self.interpret(fn, [])
 
+    def test_writebarrier_before_copy_manually_copy_card_bits(self):
+        S = lltype.GcStruct('S', ('x', lltype.Char))
+        TP = lltype.GcArray(lltype.Ptr(S))
+        def fn():
+            l1 = lltype.malloc(TP, 65)
+            l2 = lltype.malloc(TP, 33)
+            for i in range(65):
+                l1[i] = lltype.malloc(S)
+            l = lltype.malloc(TP, 100)
+            i = 0
+            while i < 65:
+                l[i] = l1[i]
+                i += 1
+            rgc.ll_arraycopy(l, l2, 0, 0, 33)
+            x = []
+            # force minor collect
+            t = (1, lltype.malloc(S))
+            for i in range(20):
+                x.append(t)
+            for i in range(33):
+                assert l2[i] == l[i]
+            return 0
+
+        self.interpret(fn, [])
+
     def test_stringbuilder(self):
         def fn():
             s = StringBuilder(4)

From pypy.commits at gmail.com  Thu Sep 14 04:54:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 14 Sep 2017 01:54:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Forgot to save this file
Message-ID: <59ba43dc.fb87df0a.df608.f988@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92387:982250d5ca98
Date: 2017-09-14 10:54 +0200
http://bitbucket.org/pypy/pypy/changeset/982250d5ca98/

Log:	Forgot to save this file

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1904,7 +1904,8 @@
                                 #ll_assert(cardbyte <= 1 and bytes == 0,
                                 #          "premature end of object")
                                 ll_assert(bytes == 0, "premature end of object")
-                                cardbyte = 1
+                                if interval_stop <= interval_start:
+                                    break
                             self.trace_and_drag_out_of_nursery_partial(
                                 obj, interval_start, interval_stop)
                         #

From pypy.commits at gmail.com  Thu Sep 14 05:25:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 14 Sep 2017 02:25:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: Redo f3070ec0eeb3
Message-ID: <59ba4b26.551d1c0a.66ea1.75ce@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92389:4cd277fc0b9d
Date: 2017-09-14 11:12 +0200
http://bitbucket.org/pypy/pypy/changeset/4cd277fc0b9d/

Log:	Redo f3070ec0eeb3

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,6 +22,7 @@
     can_usually_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -10,73 +10,65 @@
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_roots() ----------
 
-def _counting_rpy_root(obj, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_roots(gc):
-    gc._count_rpy = 0
-    gc.enumerate_all_roots(_counting_rpy_root, gc)
-    return gc._count_rpy
-
 def _append_rpy_root(obj, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected in the 'while' loop below
 
 def _do_append_rpy_roots(gc, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.enumerate_all_roots(_append_rpy_root, gc)
     gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_roots(gc):
-    count = _do_count_rpy_roots(gc)
-    extra = 16
+    # returns a list that may end with some NULLs
     while True:
-        result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
-        try:
-            _do_append_rpy_roots(gc, result)
-        except ValueError:
-            extra *= 3
-        else:
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * gc._totalroots_rpy
+        count = _do_append_rpy_roots(gc, result)
+        if count <= len(result):     # 'count' fits inside the list
             return result
+        count += (count // 8)
+        gc._totalroots_rpy = count + 10
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_referents() ----------
 
-def _count_rpy_referent(pointer, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_referents(gc, gcref):
-    gc._count_rpy = 0
-    gc.trace(llmemory.cast_ptr_to_adr(gcref), _count_rpy_referent, gc)
-    return gc._count_rpy
-
 def _append_rpy_referent(pointer, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
-                                          llmemory.GCREF)
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
+                                              llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected in the 'while' loop below
 
 def _do_append_rpy_referents(gc, gcref, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.trace(llmemory.cast_ptr_to_adr(gcref), _append_rpy_referent, gc)
+    gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_referents(gc, gcref):
-    count = _do_count_rpy_referents(gc, gcref)
-    result = [lltype.nullptr(llmemory.GCREF.TO)] * count
-    _do_append_rpy_referents(gc, gcref, result)
-    return result
+    # returns a list with no NULLs
+    result = []
+    while True:
+        count = _do_append_rpy_referents(gc, gcref, result)
+        if count <= len(result):     # 'count' fits inside the list
+            if count < len(result):
+                result = result[:count]
+            return result
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * count
 
 # ----------
 

From pypy.commits at gmail.com  Thu Sep 14 05:25:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 14 Sep 2017 02:25:56 -0700 (PDT)
Subject: [pypy-commit] pypy default: Redo 90a0d3659179;
 the problem was 9bd6fef43680.
Message-ID: <59ba4b24.08b41c0a.23311.38d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92388:0d880d3a1b92
Date: 2017-09-14 11:06 +0200
http://bitbucket.org/pypy/pypy/changeset/0d880d3a1b92/

Log:	Redo 90a0d3659179; the problem was 9bd6fef43680.

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -335,6 +335,7 @@
         callback2, attrname = _convert_callback_formats(callback)    # :-/
         setattr(self, attrname, arg)
         self.root_walker.walk_roots(callback2, callback2, callback2)
+        self.enum_live_with_finalizers(callback, arg)
         self.enum_pending_finalizers(callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
@@ -347,6 +348,12 @@
             i += 1
     enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        # as far as possible, enumerates the live objects with finalizers,
+        # even if they have not been detected as unreachable yet (but may be)
+        pass
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _copy_pending_finalizers_deque(self, deque, copy_fn):
         tmp = self.AddressDeque()
         while deque.non_empty():
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2522,6 +2522,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _collect_obj(self, obj, ignored):
         # Ignore pinned objects, which are the ones still in the nursery here.
         # Cache effects: don't read any flag out of 'obj' at this point.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1787,6 +1787,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     @staticmethod
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -269,22 +269,23 @@
             self.index_in_oldest = index + 1
             return result
 
-        def foreach(self, callback, arg):
+        def foreach(self, callback, arg, step=1):
             """Invoke 'callback(address, arg)' for all addresses in the deque.
             Typically, 'callback' is a bound method and 'arg' can be None.
+            If step > 1, only calls it for addresses multiple of 'step'.
             """
             chunk = self.oldest_chunk
             index = self.index_in_oldest
             while chunk is not self.newest_chunk:
                 while index < chunk_size:
                     callback(chunk.items[index], arg)
-                    index += 1
+                    index += step
                 chunk = chunk.next
-                index = 0
+                index -= chunk_size
             limit = self.index_in_newest
             while index < limit:
                 callback(chunk.items[index], arg)
-                index += 1
+                index += step
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def delete(self):
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -160,6 +160,10 @@
 
             ll.foreach(callback, 42)
             assert seen == addrs
+            seen = []
+            ll.foreach(callback, 42, step=2)
+            assert seen == addrs[::2]
+
             for a in addrs:
                 b = ll.popleft()
                 assert a == b

From pypy.commits at gmail.com  Thu Sep 14 15:42:17 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 12:42:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: win32 tests almost pass untranslated,
 still have resource leak
Message-ID: <59badb99.2483df0a.5c8bb.bdf4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92390:abca7cc0826b
Date: 2017-09-14 19:37 +0300
http://bitbucket.org/pypy/pypy/changeset/abca7cc0826b/

Log:	win32 tests almost pass untranslated, still have resource leak

diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -12,6 +12,7 @@
                                   'itertools', 'select', 'struct', 'binascii']}
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
     else:
         spaceconfig['usemodules'].append('fcntl')
 
@@ -39,6 +40,10 @@
 
 class BaseConnectionTest(object):
     def test_connection(self):
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         whandle.send_bytes("abc")
@@ -50,6 +55,10 @@
         assert obj == obj2
 
     def test_poll(self):
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         assert rhandle.poll() == False
@@ -64,6 +73,10 @@
 
     def test_read_into(self):
         import array, multiprocessing
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         obj = [1, 2.0, "hello"]
@@ -81,6 +94,7 @@
     }
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
 
     def setup_class(cls):
         if sys.platform != "win32":
@@ -109,6 +123,7 @@
     }
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
     else:
         spaceconfig['usemodules'].append('fcntl')
 

From pypy.commits at gmail.com  Thu Sep 14 15:42:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 12:42:22 -0700 (PDT)
Subject: [pypy-commit] pypy win32-fixes6: skip more vmprof-related tests on
 win32
Message-ID: <59badb9e.88b0df0a.b1640.2bd7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-fixes6
Changeset: r92393:16f25737077e
Date: 2017-09-14 22:40 +0300
http://bitbucket.org/pypy/pypy/changeset/16f25737077e/

Log:	skip more vmprof-related tests on win32

diff --git a/pypy/module/faulthandler/test/test_faulthander.py b/pypy/module/faulthandler/test/test_faulthander.py
--- a/pypy/module/faulthandler/test/test_faulthander.py
+++ b/pypy/module/faulthandler/test/test_faulthander.py
@@ -1,3 +1,7 @@
+import sys, py
+if sys.platform == 'win32':
+    py.test.skip('vmprof disabled on windows')
+
 class AppTestFaultHandler:
     spaceconfig = {
         "usemodules": ["faulthandler", "_vmprof"]
diff --git a/pypy/module/faulthandler/test/test_ztranslation.py b/pypy/module/faulthandler/test/test_ztranslation.py
--- a/pypy/module/faulthandler/test/test_ztranslation.py
+++ b/pypy/module/faulthandler/test/test_ztranslation.py
@@ -1,3 +1,7 @@
+import sys, py
+if sys.platform == 'win32':
+    py.test.skip('vmprof disabled on windows')
+
 from pypy.objspace.fake.checkmodule import checkmodule
 
 def test_faulthandler_translates():

From pypy.commits at gmail.com  Thu Sep 14 15:42:18 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 12:42:18 -0700 (PDT)
Subject: [pypy-commit] pypy default: fixes for C89 (win32)
Message-ID: <59badb9a.64a8df0a.a8f31.0774@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92391:9edce20cd8a3
Date: 2017-09-14 21:26 +0300
http://bitbucket.org/pypy/pypy/changeset/9edce20cd8a3/

Log:	fixes for C89 (win32)

diff --git a/pypy/module/cpyext/test/issue2482.c b/pypy/module/cpyext/test/issue2482.c
--- a/pypy/module/cpyext/test/issue2482.c
+++ b/pypy/module/cpyext/test/issue2482.c
@@ -18,9 +18,9 @@
 PyObject *make_object_base_type(void) {
 
     PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
-    if (!heap_type) return NULL;
 
     PyTypeObject *type = &heap_type->ht_type;
+    if (!heap_type) return NULL;
     type->tp_name = name;
 #ifdef ISSUE_2482
     type->tp_base = &PyBaseObject_Type; /*fails */
@@ -85,16 +85,19 @@
 #else
     PyObject *module = Py_InitModule("issue2482", issue2482_functions);
 #endif
+    PyHeapTypeObject *heap_type;
+    PyTypeObject *type;
+    PyObject * base;
     if (module == NULL)
         INITERROR;
 
-    PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
+    heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
     if (!heap_type) INITERROR;
 
-    PyTypeObject *type = &heap_type->ht_type;
+    type = &heap_type->ht_type;
     type->tp_name = name;
 
-    PyObject *base = make_object_base_type();
+    base = make_object_base_type();
     if (! base) INITERROR;
     Py_INCREF(base);
     type->tp_base = (PyTypeObject *) base;
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -340,21 +340,26 @@
         module = self.import_extension('foo', [
             ("call_recursive", "METH_NOARGS",
              """
-                int res = 0;
-                int recurse(void) {
-                    if (Py_EnterRecursiveCall(" while calling recurse"))
-                        return -1;
-                    res ++;
-                    return recurse();
-                }
-                int oldlimit = Py_GetRecursionLimit();
+                int oldlimit;
+                int recurse(void);
+                res = 0;
+                oldlimit = Py_GetRecursionLimit();
                 Py_SetRecursionLimit(200);
                 res = recurse();
                 Py_SetRecursionLimit(oldlimit);
                 if (PyErr_Occurred())
                     return NULL;
                 return PyLong_FromLong(res);
-             """),], prologue= ''' int recurse(void); '''
+             """),], prologue= '''
+                int res;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse")) {
+                        return -1;
+                    }
+                    res ++;
+                    return recurse();
+                };
+             '''
             )
         try:
             res = module.call_recursive()
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -190,9 +190,10 @@
                     0,                  /* tp_basicsize*/
                     0                  /* tp_itemsize */
                 };
+                PyObject * mod;
+                PyObject * dt;
             ''', more_init='''
-                PyObject * mod = PyImport_ImportModule("datetime");
-                PyObject * dt;
+                mod = PyImport_ImportModule("datetime");
                 if (mod == NULL) INITERROR;
                 dt = PyString_FromString("datetime");
                 datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod, dt); 

From pypy.commits at gmail.com  Thu Sep 14 15:42:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 12:42:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: getsockopt calls on win32 with BOOL
 param return 1 byte
Message-ID: <59badb9c.c55c1c0a.725e1.7410@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92392:7f45abeb0620
Date: 2017-09-14 22:34 +0300
http://bitbucket.org/pypy/pypy/changeset/7f45abeb0620/

Log:	getsockopt calls on win32 with BOOL param return 1 byte

diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -589,7 +589,12 @@
         import _socket
         s = _socket.socket()
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
-        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        ret = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2)
+        if len(ret) == 1:
+            # win32 returns a byte-as-bool
+            assert ret == b'\x00'
+        else:
+            assert ret == b'\x00\x00'
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
@@ -599,7 +604,11 @@
         import _socket
         s = _socket.socket()
         buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
-        assert buf == b'\x00' * 4
+        if len(buf) == 1:
+            # win32 returns a byte-as-bool
+            assert buf == b'\x00'
+        else:
+            assert buf == b'\x00' * 4
         raises(_socket.error, s.getsockopt,
                _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
         raises(_socket.error, s.getsockopt,
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -845,6 +845,8 @@
     @jit.dont_look_inside
     def getsockopt_int(self, level, option):
         flag_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        # some win32 calls use only a byte to represent a bool
+        flag_p[0] = 0
         try:
             flagsize_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw')
             try:

From pypy.commits at gmail.com  Thu Sep 14 15:42:24 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 12:42:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: os has no uname on win32
Message-ID: <59badba0.8e8bdf0a.f151c.ad6a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92394:1eac75e69a12
Date: 2017-09-14 22:41 +0300
http://bitbucket.org/pypy/pypy/changeset/1eac75e69a12/

Log:	os has no uname on win32

diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -329,7 +329,7 @@
     }
 
     import os
-    if os.uname()[4] == 's390x':
+    if hasattr(os, 'uname') and os.uname()[4] == 's390x':
         py.test.skip("build bot for s390x cannot open sockets")
 
     def w_make_server(self):

From pypy.commits at gmail.com  Thu Sep 14 15:42:33 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 14 Sep 2017 12:42:33 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Remove obsolete header intobject.h
Message-ID: <59badba9.06aadf0a.20936.79a1@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92395:49d988bc50f2
Date: 2017-09-14 20:42 +0100
http://bitbucket.org/pypy/pypy/changeset/49d988bc50f2/

Log:	Remove obsolete header intobject.h

diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -114,7 +114,6 @@
 #include "descrobject.h"
 #include "tupleobject.h"
 #include "dictobject.h"
-#include "intobject.h"
 #include "longobject.h"
 #include "listobject.h"
 #include "longobject.h"
diff --git a/pypy/module/cpyext/include/intobject.h b/pypy/module/cpyext/include/intobject.h
deleted file mode 100644
--- a/pypy/module/cpyext/include/intobject.h
+++ /dev/null
@@ -1,18 +0,0 @@
-
-/* Int object interface */
-
-#ifndef Py_INTOBJECT_H
-#define Py_INTOBJECT_H
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-typedef struct {
-    PyObject_HEAD
-    long ob_ival;
-} PyIntObject;
-
-#ifdef __cplusplus
-}
-#endif
-#endif /* !Py_INTOBJECT_H */

From pypy.commits at gmail.com  Thu Sep 14 16:30:41 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 13:30:41 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: sync with pypy2 - remove faulthandler
 from win32 until vmprof is fixed
Message-ID: <59bae6f1.d05b1c0a.c29ba.5ede@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92396:bb2e3674c3ee
Date: 2017-09-14 23:29 +0300
http://bitbucket.org/pypy/pypy/changeset/bb2e3674c3ee/

Log:	sync with pypy2 - remove faulthandler from win32 until vmprof is
	fixed

diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -73,6 +73,8 @@
 
     if "_cppyy" in working_modules:
         working_modules.remove("_cppyy")  # not tested on win32
+    if "faulthandler" in working_modules:
+        working_modules.remove("faulthandler")  # missing details
     if "_vmprof" in working_modules:
         working_modules.remove("_vmprof")  # FIXME: missing details
 

From pypy.commits at gmail.com  Thu Sep 14 16:35:05 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 14 Sep 2017 13:35:05 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: compatibility w/cpython and SUBCLASS flags
Message-ID: <59bae7f9.d1d81c0a.57964.759e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92397:385eb4874961
Date: 2017-09-14 23:34 +0300
http://bitbucket.org/pypy/pypy/changeset/385eb4874961/

Log:	compatibility w/cpython and SUBCLASS flags

diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -241,7 +241,7 @@
     PyBufferProcs *tp_as_buffer;
 
     /* Flags to define presence of optional/expanded features */
-    long tp_flags;
+    unsigned long tp_flags;
 
     const char *tp_doc; /* Documentation string */
 

From pypy.commits at gmail.com  Thu Sep 14 16:39:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 14 Sep 2017 13:39:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Improve compatibility of cpyext method
 descriptors with CPython.
Message-ID: <59bae90c.f781df0a.c1dd0.8d15@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92398:c2b34a5750cb
Date: 2017-09-14 21:38 +0100
http://bitbucket.org/pypy/pypy/changeset/c2b34a5750cb/

Log:	Improve compatibility of cpyext method descriptors with CPython.

	In particular, turn some misuses of them into exceptions rather than
	segfaults.

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -109,9 +109,27 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        return self.getrepr(self.space,
-                            "built-in method '%s' of '%s' object" %
-                            (self.name, self.w_objclass.getname(self.space)))
+        return self.space.newtext("<method '%s' of '%s' objects>" % (
+            self.name, self.w_objclass.name))
+
+    def descr_call(self, space, __args__):
+        args_w, kw_w = __args__.unpack()
+        if len(args_w) < 1:
+            raise oefmt(space.w_TypeError,
+                "descriptor '%s' of '%s' object needs an argument",
+                self.name, self.w_objclass.name)
+        w_instance = args_w[0]
+        # XXX: needs a stricter test
+        if not space.isinstance_w(w_instance, self.w_objclass):
+            raise oefmt(space.w_TypeError,
+                "descriptor '%s' requires a '%s' object but received a '%T'",
+                self.name, self.w_objclass.name, w_instance)
+        w_args = space.newtuple(args_w[1:])
+        w_kw = space.newdict()
+        for key, w_obj in kw_w.items():
+            space.setitem(w_kw, space.newtext(key), w_obj)
+        ret = self.call(space, w_instance, w_args, w_kw)
+        return ret
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
@@ -207,7 +225,7 @@
     ret = self.call(space, None, w_args, w_kw)
     return ret
 
-def cmethod_descr_call(space, w_self, __args__):
+def cclassmethod_descr_call(space, w_self, __args__):
     self = space.interp_w(W_PyCFunctionObject, w_self)
     args_w, kw_w = __args__.unpack()
     if len(args_w) < 1:
@@ -248,9 +266,9 @@
 W_PyCFunctionObject.typedef.acceptable_as_base_class = False
 
 W_PyCMethodObject.typedef = TypeDef(
-    'method',
+    'method_descriptor',
     __get__ = interp2app(cmethod_descr_get),
-    __call__ = interp2app(cmethod_descr_call),
+    __call__ = interp2app(W_PyCMethodObject.descr_call),
     __name__ = interp_attrproperty('name', cls=W_PyCMethodObject,
         wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCMethodObject),
@@ -261,7 +279,7 @@
 W_PyCClassMethodObject.typedef = TypeDef(
     'classmethod',
     __get__ = interp2app(cclassmethod_descr_get),
-    __call__ = interp2app(cmethod_descr_call),
+    __call__ = interp2app(cclassmethod_descr_call),
     __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject,
         wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass',
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -122,6 +122,14 @@
         obj = module.fooType.classmeth()
         assert obj is module.fooType
 
+    def test_methoddescr(self):
+        module = self.import_module(name='foo')
+        descr = module.fooType.copy
+        assert type(descr).__name__ == 'method_descriptor'
+        assert str(descr) == "<method 'copy' of 'foo.foo' objects>"
+        assert repr(descr) == "<method 'copy' of 'foo.foo' objects>"
+        raises(TypeError, descr, None)
+
     def test_new(self):
         # XXX cpython segfaults but if run singly (with -k test_new) this passes
         module = self.import_module(name='foo')
@@ -1252,13 +1260,13 @@
                   ((PyHeapTypeObject*)Base2)->ht_name = dummyname;
                   ((PyHeapTypeObject*)Base12)->ht_name = dummyname;
                 }
-                #endif 
-                #endif 
+                #endif
+                #endif
                 Base1->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
                 Base2->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
                 Base12->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE;
                 Base12->tp_base = Base1;
-                Base12->tp_bases = PyTuple_Pack(2, Base1, Base2); 
+                Base12->tp_bases = PyTuple_Pack(2, Base1, Base2);
                 Base12->tp_doc = "The Base12 type or object";
                 if (PyType_Ready(Base1) < 0) return NULL;
                 if (PyType_Ready(Base2) < 0) return NULL;
@@ -1426,4 +1434,4 @@
             pass
         assert module.test_flags(MyList, Py_TPFLAGS_LIST_SUBCLASS) == 0
 
-         
+

From pypy.commits at gmail.com  Thu Sep 14 16:42:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 14 Sep 2017 13:42:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the comment.
Message-ID: <59bae9c9.1986df0a.cb8d5.b82c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3008:86653ca1a0cf
Date: 2017-09-14 22:42 +0200
http://bitbucket.org/cffi/cffi/changeset/86653ca1a0cf/

Log:	Update the comment.

diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -234,9 +234,10 @@
     save_errno();
 
     /* We need the infotuple here.  We could always go through
-       interp->modules['..'][externpy], but to avoid the extra dict
+       _update_cache_to_call_python(), but to avoid the extra dict
        lookups, we cache in (reserved1, reserved2) the last seen pair
-       (interp->modules, infotuple).
+       (interp->modules, infotuple).  The first item in this tuple is
+       a random PyObject that identifies the subinterpreter.
     */
     if (externpy->reserved1 == NULL) {
         /* Not initialized!  We didn't call @ffi.def_extern() on this

From pypy.commits at gmail.com  Fri Sep 15 03:57:53 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 00:57:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix issue 2657,
 but why is this even an exported API function?
Message-ID: <59bb8801.90d71c0a.12690.664e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92399:d733a59858a7
Date: 2017-09-15 10:55 +0300
http://bitbucket.org/pypy/pypy/changeset/d733a59858a7/

Log:	fix issue 2657, but why is this even an exported API function?

diff --git a/pypy/module/cpyext/include/unicodeobject.h b/pypy/module/cpyext/include/unicodeobject.h
--- a/pypy/module/cpyext/include/unicodeobject.h
+++ b/pypy/module/cpyext/include/unicodeobject.h
@@ -340,7 +340,7 @@
 
 Py_LOCAL_INLINE(size_t) Py_UNICODE_strlen(const Py_UNICODE *u)
 {
-    int res = 0;
+    size_t res = 0;
     while(*u++)
         res++;
     return res;

From pypy.commits at gmail.com  Fri Sep 15 04:02:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 01:02:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: remove redundant test, fix translation
Message-ID: <59bb891b.d7a3500a.76d6c.1ad7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92400:921cf6b61598
Date: 2017-09-15 11:01 +0300
http://bitbucket.org/pypy/pypy/changeset/921cf6b61598/

Log:	remove redundant test, fix translation

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -81,3 +81,7 @@
 .. branch: pycheck-macros
 
 Convert many Py*_Check cpyext functions into macros, like CPython.
+
+.. branch: py_ssize_t
+
+Explicitly use Py_ssize_t as the Signed type in pypy c-api
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -110,20 +110,20 @@
 
     def descr_method_repr(self):
         return self.space.newtext("<method '%s' of '%s' objects>" % (
-            self.name, self.w_objclass.name))
+            self.name, self.w_objclass.getname(self.space)))
 
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' of '%s' object needs an argument",
-                self.name, self.w_objclass.name)
+                self.name, self.w_objclass.getname(self.space))
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' requires a '%s' object but received a '%T'",
-                self.name, self.w_objclass.name, w_instance)
+                self.name, self.w_objclass.getname(self.space), w_instance)
         w_args = space.newtuple(args_w[1:])
         w_kw = space.newdict()
         for key, w_obj in kw_w.items():
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -117,25 +117,3 @@
         assert mod.check(A.meth) == 0
         assert mod.check(A.stat) == 0
  
-
-class TestPyCMethodObject(BaseApiTest):
-    def test_repr(self, space, api):
-        """
-        W_PyCMethodObject has a repr string which describes it as a method
-        and gives its name and the name of its class.
-        """
-        def func(space, w_self, w_args):
-            return space.w_None
-        c_func = ApiFunction([PyObject, PyObject], PyObject, func)
-        func.api_func = c_func
-        ml = lltype.malloc(PyMethodDef, flavor='raw', zero=True)
-        namebuf = rffi.cast(rffi.CONST_CCHARP, rffi.str2charp('func'))
-        ml.c_ml_name = namebuf
-        ml.c_ml_meth = rffi.cast(PyCFunction, c_func.get_llhelper(space))
-
-        method = api.PyDescr_NewMethod(space.w_bytes, ml)
-        assert repr(method).startswith(
-            "<built-in method 'func' of 'str' object ")
-
-        rffi.free_charp(namebuf)
-        lltype.free(ml, flavor='raw')
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -846,7 +846,8 @@
     def getsockopt_int(self, level, option):
         flag_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
         # some win32 calls use only a byte to represent a bool
-        flag_p[0] = 0
+        # zero out so the result is correct anyway
+        flag_p[0] = rffi.cast(rffi.INT, 0)
         try:
             flagsize_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw')
             try:

From pypy.commits at gmail.com  Fri Sep 15 05:49:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 02:49:31 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <59bba22b.44231c0a.a77a0.1f5a@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92401:2ec9c954ee0a
Date: 2017-09-15 12:46 +0300
http://bitbucket.org/pypy/pypy/changeset/2ec9c954ee0a/

Log:	fix translation

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -943,7 +943,7 @@
     res = PyType_GenericAlloc(space, space.w_type, 0)
     res = cts.cast('PyHeapTypeObject *', res)
     typ = res.c_ht_type
-    typ.c_tp_flags = rffi.cast(lltype.Signed, spec.c_flags)
+    typ.c_tp_flags = rffi.cast(lltype.Unsigned, spec.c_flags)
     typ.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
     specname = rffi.charp2str(cts.cast('char*', spec.c_name))
     dotpos = specname.rfind('.')

From pypy.commits at gmail.com  Fri Sep 15 05:56:09 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 02:56:09 -0700 (PDT)
Subject: [pypy-commit] buildbot cleanup-hg-bookmarks: remove link to numpy
 results from page top, add link to py3.5 instead
Message-ID: <59bba3b9.cad81c0a.e1693.6529@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cleanup-hg-bookmarks
Changeset: r1026:06d6f0d7b2c2
Date: 2017-09-15 12:55 +0300
http://bitbucket.org/pypy/buildbot/changeset/06d6f0d7b2c2/

Log:	remove link to numpy results from page top, add link to py3.5
	instead

diff --git a/master/templates/layout.html b/master/templates/layout.html
--- a/master/templates/layout.html
+++ b/master/templates/layout.html
@@ -25,8 +25,9 @@
         <a href="{{ path_to_root or '.' }}">Home</a>
         <!-- PyPy specific items -->
         - <a href="http://speed.pypy.org/">Speed</a>
-        - <a href="{{ path_to_root }}numpy-status/">NumPy compatibility</a>
+        - <!-- a href="{{ path_to_root }}numpy-status/">NumPy compatibility</a -->
         - <a href="{{ path_to_root }}summary?branch=&lt;trunk&gt;">Summary (trunk)</a>
+        - <a href="{{ path_to_root }}summary?branch=3.5">Summary (py3.5)</a>
         - <a href="{{ path_to_root }}summary">Summary</a>
         - <a href="{{ path_to_root }}nightly/">Nightly builds</a>
         <!-- end of PyPy specific items -->

From pypy.commits at gmail.com  Fri Sep 15 05:59:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 02:59:20 -0700 (PDT)
Subject: [pypy-commit] buildbot cleanup-hg-bookmarks: close branch to be
 merged
Message-ID: <59bba478.b399df0a.dff36.56d3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: cleanup-hg-bookmarks
Changeset: r1027:be49507c1176
Date: 2017-09-15 12:57 +0300
http://bitbucket.org/pypy/buildbot/changeset/be49507c1176/

Log:	close branch to be merged


From pypy.commits at gmail.com  Fri Sep 15 05:59:21 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 02:59:21 -0700 (PDT)
Subject: [pypy-commit] buildbot default: merge branch to remove hg bookmarks
 during hg cleanup
Message-ID: <59bba479.d9471c0a.46df2.74df@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r1028:201463d5f98a
Date: 2017-09-15 12:59 +0300
http://bitbucket.org/pypy/buildbot/changeset/201463d5f98a/

Log:	merge branch to remove hg bookmarks during hg cleanup

diff --git a/bot2/pypybuildbot/builds.py b/bot2/pypybuildbot/builds.py
--- a/bot2/pypybuildbot/builds.py
+++ b/bot2/pypybuildbot/builds.py
@@ -333,11 +333,25 @@
            workdir=workdir))
 
 def update_hg(platform, factory, repourl, workdir, use_branch,
-              force_branch=None):
+              force_branch=None, wipe_bookmarks=False):
     if not use_branch:
         assert force_branch is None
         update_hg_old_method(platform, factory, repourl, workdir)
         return
+
+    if wipe_bookmarks:
+        # We don't use bookmarks at all.  If a bookmark accidentally gets
+        # created and pushed to the server and we pull it down, it gets stuck
+        # here.  Deleting it from the server doesn't seem to delete it from
+        # the local checkout.  So, manually clean it up.
+        factory.addStep(ShellCmd(
+            description="cleanup bookmarks",
+            command=["rm", "-f", ".hg/bookmarks"] if platform != 'win32'
+               else [r"cmd /c if exist .hg\bookmarks del .hg\bookmarks"],
+            workdir=workdir,
+            haltOnFailure=False,
+        ))
+
     factory.addStep(
             Mercurial(
                 repourl=repourl,
@@ -374,7 +388,7 @@
                                   doStepIf=ParseRevision.doStepIf))
     #
     update_hg(platform, factory, repourl, workdir, use_branch=True,
-              force_branch=force_branch)
+              force_branch=force_branch, wipe_bookmarks=True)
     #
     factory.addStep(CheckGotRevision(workdir=workdir))
 
@@ -410,7 +424,7 @@
     # If target_tmpdir is empty, crash.
     tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s'
     pytest = "pytest"
-    factory.addStep(ShellCmd( 
+    factory.addStep(ShellCmd(
         description="mkdir for tests",
         command=['python', '-c', Interpolate("import os;  os.mkdir(r'" + \
                     tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \
@@ -424,7 +438,7 @@
                    '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"]
     else:
         command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime',
-                   '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] 
+                   '+' + nDays, '-exec', 'rm', '-r', '{}', ';']
     factory.addStep(SuccessAlways(
         description="cleanout old test files",
         command = command,
@@ -481,7 +495,7 @@
         # If target_tmpdir is empty, crash.
         tmp_or_crazy = '%(prop:target_tmpdir:-crazy/name/so/mkdir/fails/)s'
         pytest = "pytest"
-        self.addStep(ShellCmd( 
+        self.addStep(ShellCmd(
             description="mkdir for tests",
             command=['python', '-c', Interpolate("import os;  os.mkdir(r'" + \
                         tmp_or_crazy + pytest + "') if not os.path.exists(r'" + \
@@ -495,7 +509,7 @@
                        '/D', '-' + nDays, '/c', "cmd /c rmdir /q /s @path"]
         else:
             command = ['find', Interpolate(tmp_or_crazy + pytest), '-mtime',
-                       '+' + nDays, '-exec', 'rm', '-r', '{}', ';'] 
+                       '+' + nDays, '-exec', 'rm', '-r', '{}', ';']
         self.addStep(SuccessAlways(
             description="cleanout old test files",
             command = command,
@@ -976,7 +990,7 @@
             workdir='pypy-c',
             haltOnFailure=True,
             ))
-        
+
         if platform == 'win32':
             self.addStep(ShellCmd(
                 description='move decompressed dir',
@@ -1010,7 +1024,7 @@
         # obtain a pypy-compatible branch of numpy
         numpy_url = 'https://www.bitbucket.org/pypy/numpy'
         update_git(platform, self, numpy_url, 'numpy_src', branch='master',
-                   alwaysUseLatest=True, # ignore pypy rev number when 
+                   alwaysUseLatest=True, # ignore pypy rev number when
                                          # triggered by a pypy build
                    )
 
diff --git a/master/templates/layout.html b/master/templates/layout.html
--- a/master/templates/layout.html
+++ b/master/templates/layout.html
@@ -25,8 +25,9 @@
         <a href="{{ path_to_root or '.' }}">Home</a>
         <!-- PyPy specific items -->
         - <a href="http://speed.pypy.org/">Speed</a>
-        - <a href="{{ path_to_root }}numpy-status/">NumPy compatibility</a>
+        - <!-- a href="{{ path_to_root }}numpy-status/">NumPy compatibility</a -->
         - <a href="{{ path_to_root }}summary?branch=&lt;trunk&gt;">Summary (trunk)</a>
+        - <a href="{{ path_to_root }}summary?branch=3.5">Summary (py3.5)</a>
         - <a href="{{ path_to_root }}summary">Summary</a>
         - <a href="{{ path_to_root }}nightly/">Nightly builds</a>
         <!-- end of PyPy specific items -->

From pypy.commits at gmail.com  Fri Sep 15 10:39:50 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 15 Sep 2017 07:39:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: argument pbyteorder should be precisely
 int*
Message-ID: <59bbe636.12881c0a.d7a39.6140@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92402:5574c7edd8e2
Date: 2017-09-15 14:02 +0000
http://bitbucket.org/pypy/pypy/changeset/5574c7edd8e2/

Log:	argument pbyteorder should be precisely int*

diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2355,12 +2355,14 @@
             }
         return ret;
     }
+    /*
     else if(obj1->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name);
     else if(obj2->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name);
+    */
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;
 }
@@ -2409,12 +2411,14 @@
         Py_DECREF(lhs);
         return ret;
     }
+    /*
     else if(obj1->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name);
     else if(obj2->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name);
+    */
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;
 }
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -24,6 +24,7 @@
 cts.parse_header(parse_dir / 'cpyext_unicodeobject.h')
 PyUnicodeObject = cts.gettype('PyUnicodeObject*')
 Py_UNICODE = cts.gettype('Py_UNICODE')
+INT_realP = lltype.Ptr(lltype.Array(rffi.INT_real, hints={'nolength': True}))
 
 @bootstrap_function
 def init_unicodeobject(space):
@@ -730,7 +731,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INT_realP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
@@ -780,11 +781,11 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INT_realP], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-32 encoded buffer string and
     return the corresponding Unicode object.  errors (if non-NULL)
@@ -836,7 +837,7 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 

From pypy.commits at gmail.com  Fri Sep 15 11:38:14 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 08:38:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <59bbf3e6.138a1c0a.ea9e.a2a6@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92404:3bf25e792a8e
Date: 2017-09-15 16:37 +0100
http://bitbucket.org/pypy/pypy/changeset/3bf25e792a8e/

Log:	merge heads

diff --git a/pypy/module/cpyext/test/array.c b/pypy/module/cpyext/test/array.c
--- a/pypy/module/cpyext/test/array.c
+++ b/pypy/module/cpyext/test/array.c
@@ -2355,12 +2355,14 @@
             }
         return ret;
     }
+    /*
     else if(obj1->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name);
     else if(obj2->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name);
+    */
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;
 }
@@ -2409,12 +2411,14 @@
         Py_DECREF(lhs);
         return ret;
     }
+    /*
     else if(obj1->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj1)->ob_descr->typecode, obj2->ob_type->tp_name);
     else if(obj2->ob_type == &Arraytype)
         fprintf(stderr, "\nCannot multiply array of type %c and %s\n",
             ((arrayobject*)obj2)->ob_descr->typecode, obj1->ob_type->tp_name);
+    */
     Py_INCREF(Py_NotImplemented);
     return Py_NotImplemented;
 }
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -24,6 +24,7 @@
 cts.parse_header(parse_dir / 'cpyext_unicodeobject.h')
 PyUnicodeObject = cts.gettype('PyUnicodeObject*')
 Py_UNICODE = cts.gettype('Py_UNICODE')
+INT_realP = lltype.Ptr(lltype.Array(rffi.INT_real, hints={'nolength': True}))
 
 @bootstrap_function
 def init_unicodeobject(space):
@@ -730,7 +731,7 @@
 if sys.platform == 'win32':
     make_conversion_functions('MBCS', 'mbcs')
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INT_realP], PyObject)
 def PyUnicode_DecodeUTF16(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-16 encoded buffer string and return the
     corresponding Unicode object.  errors (if non-NULL) defines the error
@@ -780,11 +781,11 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 
- at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, rffi.INTP], PyObject)
+ at cpython_api([CONST_STRING, Py_ssize_t, CONST_STRING, INT_realP], PyObject)
 def PyUnicode_DecodeUTF32(space, s, size, llerrors, pbyteorder):
     """Decode length bytes from a UTF-32 encoded buffer string and
     return the corresponding Unicode object.  errors (if non-NULL)
@@ -836,7 +837,7 @@
         None, # errorhandler
         byteorder)
     if pbyteorder is not None:
-        pbyteorder[0] = rffi.cast(rffi.INT, byteorder)
+        pbyteorder[0] = rffi.cast(rffi.INT_real, byteorder)
 
     return space.newunicode(result)
 

From pypy.commits at gmail.com  Fri Sep 15 11:38:12 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 08:38:12 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59bbf3e4.f1b8df0a.e9a34.899c@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92403:3d6ac44c6fff
Date: 2017-09-15 16:37 +0100
http://bitbucket.org/pypy/pypy/changeset/3d6ac44c6fff/

Log:	hg merge default

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -81,3 +81,7 @@
 .. branch: pycheck-macros
 
 Convert many Py*_Check cpyext functions into macros, like CPython.
+
+.. branch: py_ssize_t
+
+Explicitly use Py_ssize_t as the Signed type in pypy c-api
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -114,12 +114,15 @@
 INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
 
 
-Abridged method (for -Ojit builds using Visual Studio 2008)
------------------------------------------------------------
+Abridged method (using Visual Studio 2008)
+------------------------------------------
 
 Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_59.zip
+(for post-5.8 builds) with sha256 checksum
+``0f96c045db1f5f73ad0fae7857caa69c261324bd8e51f6d2ad1fa842c4a5f26f``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(for post-5.7.1 builds) with sha256 checksum 
+(to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
@@ -135,8 +138,8 @@
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
 
-Nonabrided method (building from scratch)
------------------------------------------
+Nonabridged method (building from scratch)
+------------------------------------------
 
 If you want to, you can rebuild everything from scratch by continuing.
 
@@ -209,17 +212,85 @@
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
-Download the source code of expat on sourceforge:
-https://github.com/libexpat/libexpat/releases and extract it in the base directory.
-Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
-with Visual Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for
-the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes)
+CPython compiles expat from source as part of the build. PyPy uses the same
+code base, but expects to link to a static lib of expat. Here are instructions
+to reproduce the static lib in version 2.2.4.
 
-Then, copy the file ``win32\bin\release\libexpat.lib`` into
-LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in
-INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH.
+Download the source code of expat: https://github.com/libexpat/libexpat. 
+``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
+``vcvars.bat`` to set up the visual compiler tools, and CD into the source
+directory. Create a file ``stdbool.h`` with the content
+
+.. code-block:: c
+
+    #pragma once
+
+    #define false   0
+    #define true    1
+
+    #define bool int
+
+and put it in a place on the ``INCLUDE`` path, or create it in the local
+directory and add ``.`` to the ``INCLUDE`` path::
+
+    SET INCLUDE=%INCLUDE%;.
+
+Then compile all the ``*.c`` file into ``*.obj``::
+
+    cl.exe /nologo /MD  /O2 *c /c
+    rem for debug
+    cl.exe /nologo /MD  /O0 /Ob0 /Zi *c /c
+
+You may need to move some variable declarations to the beginning of the
+function, to be compliant with C89 standard. Here is the diff for version 2.2.4
+
+.. code-block:: diff
+
+    diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
+    index 007aed0..a2dcaad 100644
+    --- a/expat/lib/xmltok.c
+    +++ b/expat/lib/xmltok.c
+    @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
+       /* Avoid copying partial characters (due to limited space). */
+       const ptrdiff_t bytesAvailable = fromLim - *fromP;
+       const ptrdiff_t bytesStorable = toLim - *toP;
+    +  const char * fromLimBefore;
+    +  ptrdiff_t bytesToCopy;
+       if (bytesAvailable > bytesStorable) {
+         fromLim = *fromP + bytesStorable;
+         output_exhausted = true;
+       }
+
+       /* Avoid copying partial characters (from incomplete input). */
+    -  const char * const fromLimBefore = fromLim;
+    +  fromLimBefore = fromLim;
+       align_limit_to_full_utf8_characters(*fromP, &fromLim);
+       if (fromLim < fromLimBefore) {
+         input_incomplete = true;
+       }
+
+    -  const ptrdiff_t bytesToCopy = fromLim - *fromP;
+    +  bytesToCopy = fromLim - *fromP;
+       memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
+       *fromP += bytesToCopy;
+       *toP += bytesToCopy;
+
+
+Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
+
+    cl /LD *.obj libexpat.def /Felibexpat.dll 
+    rem for debug
+    rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
+
+    rem this will override the export library created in the step above
+    rem but tests do not need the export library, they load the dll dynamically
+    lib *.obj /out:libexpat.lib
+
+Then, copy 
+
+- ``libexpat.lib`` into LIB
+- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
+- ``libexpat.dll`` into PATH
 
 
 The OpenSSL library
@@ -363,7 +434,7 @@
 It is probably not too much work if the goal is only to get a translated
 PyPy executable, and to run all tests before translation.  But you need
 to start somewhere, and you should start with some tests in
-rpython/translator/c/test/, like ``test_standalone.py`` and
+``rpython/translator/c/test/``, like ``test_standalone.py`` and
 ``test_newgc.py``: try to have them pass on top of CPython64/64.
 
 Keep in mind that this runs small translations, and some details may go
@@ -373,7 +444,7 @@
 should be something like ``long long``.
 
 What is more generally needed is to review all the C files in
-rpython/translator/c/src for the word ``long``, because this means a
+``rpython/translator/c/src`` for the word ``long``, because this means a
 32-bit integer even on Win64.  Replace it with ``Signed`` most of the
 times.  You can replace one with the other without breaking anything on
 any other platform, so feel free to.
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -36,7 +36,7 @@
         return space.int_w(w_src)
     except OperationError as e:
         if e.match(space, space.w_TypeError):
-            raise oefmt(space.w_TypeError, '"%s" must be a string', attrname)
+            raise oefmt(space.w_TypeError, '"%s" must be an int', attrname)
         raise
 
 def _get_str(space, w_src, default, attrname):
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -521,7 +521,12 @@
         import _socket
         s = _socket.socket()
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
-        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        ret = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2)
+        if len(ret) == 1:
+            # win32 returns a byte-as-bool
+            assert ret == b'\x00'
+        else:
+            assert ret == b'\x00\x00'
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
@@ -531,7 +536,11 @@
         import _socket
         s = _socket.socket()
         buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
-        assert buf == b'\x00' * 4
+        if len(buf) == 1:
+            # win32 returns a byte-as-bool
+            assert buf == b'\x00'
+        else:
+            assert buf == b'\x00' * 4
         raises(_socket.error, s.getsockopt,
                _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
         raises(_socket.error, s.getsockopt,
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1360,6 +1360,7 @@
                          source_dir / "mysnprintf.c",
                          source_dir / "pythonrun.c",
                          source_dir / "sysmodule.c",
+                         source_dir / "complexobject.c",
                          source_dir / "cobject.c",
                          source_dir / "structseq.c",
                          source_dir / "capsule.c",
@@ -1368,7 +1369,6 @@
                          source_dir / "missing.c",
                          source_dir / "pymem.c",
                          source_dir / "bytesobject.c",
-                         source_dir / "complexobject.c",
                          source_dir / "import.c",
                          source_dir / "_warnings.c",
                          source_dir / "pylifecycle.c",
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -102,9 +102,27 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        return self.getrepr(
-            self.space, u"built-in method '%s' of '%s' object" %
-            (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
+        return self.space.newtext("<method '%s' of '%s' objects>" % (
+            self.name, self.w_objclass.getname(self.space).encode('utf-8')))
+
+    def descr_call(self, space, __args__):
+        args_w, kw_w = __args__.unpack()
+        if len(args_w) < 1:
+            raise oefmt(space.w_TypeError,
+                "descriptor '%8' of '%N' object needs an argument",
+                self.name, self.w_objclass)
+        w_instance = args_w[0]
+        # XXX: needs a stricter test
+        if not space.isinstance_w(w_instance, self.w_objclass):
+            raise oefmt(space.w_TypeError,
+                "descriptor '%8' requires a '%N' object but received a '%T'",
+                self.name, self.w_objclass, w_instance)
+        w_args = space.newtuple(args_w[1:])
+        w_kw = space.newdict()
+        for key, w_obj in kw_w.items():
+            space.setitem(w_kw, space.newtext(key), w_obj)
+        ret = self.call(space, w_instance, w_args, w_kw)
+        return ret
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
@@ -200,7 +218,7 @@
     ret = self.call(space, None, w_args, w_kw)
     return ret
 
-def cmethod_descr_call(space, w_self, __args__):
+def cclassmethod_descr_call(space, w_self, __args__):
     self = space.interp_w(W_PyCFunctionObject, w_self)
     args_w, kw_w = __args__.unpack()
     if len(args_w) < 1:
@@ -238,9 +256,9 @@
 W_PyCFunctionObject.typedef.acceptable_as_base_class = False
 
 W_PyCMethodObject.typedef = TypeDef(
-    'method',
+    'method_descriptor',
     __get__ = interp2app(cmethod_descr_get),
-    __call__ = interp2app(cmethod_descr_call),
+    __call__ = interp2app(W_PyCMethodObject.descr_call),
     __name__ = interp_attrproperty('name', cls=W_PyCMethodObject,
         wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass', cls=W_PyCMethodObject),
@@ -251,7 +269,7 @@
 W_PyCClassMethodObject.typedef = TypeDef(
     'classmethod',
     __get__ = interp2app(cclassmethod_descr_get),
-    __call__ = interp2app(cmethod_descr_call),
+    __call__ = interp2app(cclassmethod_descr_call),
     __name__ = interp_attrproperty('name', cls=W_PyCClassMethodObject,
         wrapfn="newtext_or_none"),
     __objclass__ = interp_attrproperty_w('w_objclass',
diff --git a/pypy/module/cpyext/test/issue2482.c b/pypy/module/cpyext/test/issue2482.c
--- a/pypy/module/cpyext/test/issue2482.c
+++ b/pypy/module/cpyext/test/issue2482.c
@@ -18,9 +18,9 @@
 PyObject *make_object_base_type(void) {
 
     PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
-    if (!heap_type) return NULL;
 
     PyTypeObject *type = &heap_type->ht_type;
+    if (!heap_type) return NULL;
     type->tp_name = name;
 #ifdef ISSUE_2482
     type->tp_base = &PyBaseObject_Type; /*fails */
@@ -85,16 +85,19 @@
 #else
     PyObject *module = Py_InitModule("issue2482", issue2482_functions);
 #endif
+    PyHeapTypeObject *heap_type;
+    PyTypeObject *type;
+    PyObject * base;
     if (module == NULL)
         INITERROR;
 
-    PyHeapTypeObject *heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
+    heap_type = (PyHeapTypeObject *) PyType_Type.tp_alloc(&PyType_Type, 0);
     if (!heap_type) INITERROR;
 
-    PyTypeObject *type = &heap_type->ht_type;
+    type = &heap_type->ht_type;
     type->tp_name = name;
 
-    PyObject *base = make_object_base_type();
+    base = make_object_base_type();
     if (! base) INITERROR;
     Py_INCREF(base);
     type->tp_base = (PyTypeObject *) base;
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -348,21 +348,26 @@
         module = self.import_extension('foo', [
             ("call_recursive", "METH_NOARGS",
              """
-                int res = 0;
-                int recurse(void) {
-                    if (Py_EnterRecursiveCall(" while calling recurse"))
-                        return -1;
-                    res ++;
-                    return recurse();
-                }
-                int oldlimit = Py_GetRecursionLimit();
+                int oldlimit;
+                int recurse(void);
+                res = 0;
+                oldlimit = Py_GetRecursionLimit();
                 Py_SetRecursionLimit(200);
                 res = recurse();
                 Py_SetRecursionLimit(oldlimit);
                 if (PyErr_Occurred())
                     return NULL;
                 return PyLong_FromLong(res);
-             """),], prologue= ''' int recurse(void); '''
+             """),], prologue= '''
+                int res;
+                int recurse(void) {
+                    if (Py_EnterRecursiveCall(" while calling recurse")) {
+                        return -1;
+                    }
+                    res ++;
+                    return recurse();
+                };
+             '''
             )
         excinfo = raises(RecursionError, module.call_recursive)
         assert 'while calling recurse' in str(excinfo.value)
diff --git a/pypy/module/cpyext/test/test_methodobject.py b/pypy/module/cpyext/test/test_methodobject.py
--- a/pypy/module/cpyext/test/test_methodobject.py
+++ b/pypy/module/cpyext/test/test_methodobject.py
@@ -100,26 +100,3 @@
         assert mod.check(A) == 0
         assert mod.check(A.meth) == 0
         assert mod.check(A.stat) == 0
- 
-
-class TestPyCMethodObject(BaseApiTest):
-    def test_repr(self, space, api):
-        """
-        W_PyCMethodObject has a repr string which describes it as a method
-        and gives its name and the name of its class.
-        """
-        def func(space, w_self, w_args):
-            return space.w_None
-        c_func = ApiFunction([PyObject, PyObject], PyObject, func)
-        func.api_func = c_func
-        ml = lltype.malloc(PyMethodDef, flavor='raw', zero=True)
-        namebuf = rffi.cast(rffi.CONST_CCHARP, rffi.str2charp('func'))
-        ml.c_ml_name = namebuf
-        ml.c_ml_meth = rffi.cast(PyCFunction, c_func.get_llhelper(space))
-
-        method = api.PyDescr_NewMethod(space.w_unicode, ml)
-        assert repr(method).startswith(
-            "<built-in method 'func' of 'str' object ")
-
-        rffi.free_charp(namebuf)
-        lltype.free(ml, flavor='raw')
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -125,6 +125,16 @@
         obj = module.fooType.classmeth()
         assert obj is module.fooType
 
+    def test_methoddescr(self):
+        module = self.import_module(name='foo')
+        descr = module.fooType.copy
+        assert type(descr).__name__ == 'method_descriptor'
+        assert str(descr) in ("<method 'copy' of 'foo.foo' objects>",
+            "<method 'copy' of 'foo' objects>")
+        assert repr(descr) in ("<method 'copy' of 'foo.foo' objects>",
+            "<method 'copy' of 'foo' objects>")
+        raises(TypeError, descr, None)
+
     def test_new(self):
         # XXX cpython segfaults but if run singly (with -k test_new) this passes
         module = self.import_module(name='foo')
@@ -1341,9 +1351,9 @@
         module = self.import_extension('foo', [
            ("test_flags", "METH_VARARGS",
             '''
-                long in_flag, my_flag;
+                long long in_flag, my_flag;
                 PyObject * obj;
-                if (!PyArg_ParseTuple(args, "Ol", &obj, &in_flag))
+                if (!PyArg_ParseTuple(args, "OL", &obj, &in_flag))
                     return NULL;
                 if (!PyType_Check(obj))
                 {
diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -190,9 +190,10 @@
                     0,                  /* tp_basicsize*/
                     0                  /* tp_itemsize */
                 };
+                PyObject * mod1;
+                PyObject * dt;
             ''', more_init='''
-                PyObject * mod1 = PyImport_ImportModule("datetime");
-                PyObject * dt;
+                mod1 = PyImport_ImportModule("datetime");
                 if (mod1 == NULL) INITERROR;
                 dt = PyUnicode_FromString("datetime");
                 datetime_cls = (PyTypeObject*)PyObject_GetAttr(mod1, dt);
diff --git a/pypy/module/select/test/test_select.py b/pypy/module/select/test/test_select.py
--- a/pypy/module/select/test/test_select.py
+++ b/pypy/module/select/test/test_select.py
@@ -330,7 +330,7 @@
     }
 
     import os
-    if os.uname()[4] == 's390x':
+    if hasattr(os, 'uname') and os.uname()[4] == 's390x':
         py.test.skip("build bot for s390x cannot open sockets")
 
     def w_make_server(self):
diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -22,6 +22,7 @@
     can_usually_pin_objects = False
     object_minimal_size = 0
     gcflag_extra = 0   # or a real GC flag that is always 0 when not collecting
+    _totalroots_rpy = 0   # for inspector.py
 
     def __init__(self, config, chunk_size=DEFAULT_CHUNK_SIZE,
                  translated_to_c=True):
@@ -335,6 +336,7 @@
         callback2, attrname = _convert_callback_formats(callback)    # :-/
         setattr(self, attrname, arg)
         self.root_walker.walk_roots(callback2, callback2, callback2)
+        self.enum_live_with_finalizers(callback, arg)
         self.enum_pending_finalizers(callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
@@ -347,6 +349,12 @@
             i += 1
     enum_pending_finalizers._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        # as far as possible, enumerates the live objects with finalizers,
+        # even if they have not been detected as unreachable yet (but may be)
+        pass
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _copy_pending_finalizers_deque(self, deque, copy_fn):
         tmp = self.AddressDeque()
         while deque.non_empty():
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1897,8 +1897,15 @@
                         if cardbyte & 1:
                             if interval_stop > length:
                                 interval_stop = length
-                                ll_assert(cardbyte <= 1 and bytes == 0,
-                                          "premature end of object")
+                                #--- the sanity check below almost always
+                                #--- passes, except in situations like
+                                #--- test_writebarrier_before_copy_manually\
+                                #    _copy_card_bits
+                                #ll_assert(cardbyte <= 1 and bytes == 0,
+                                #          "premature end of object")
+                                ll_assert(bytes == 0, "premature end of object")
+                                if interval_stop <= interval_start:
+                                    break
                             self.trace_and_drag_out_of_nursery_partial(
                                 obj, interval_start, interval_stop)
                         #
@@ -2515,6 +2522,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     def _collect_obj(self, obj, ignored):
         # Ignore pinned objects, which are the ones still in the nursery here.
         # Cache effects: don't read any flag out of 'obj' at this point.
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -10,73 +10,65 @@
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_roots() ----------
 
-def _counting_rpy_root(obj, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_roots(gc):
-    gc._count_rpy = 0
-    gc.enumerate_all_roots(_counting_rpy_root, gc)
-    return gc._count_rpy
-
 def _append_rpy_root(obj, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(obj, llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected in the 'while' loop below
 
 def _do_append_rpy_roots(gc, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.enumerate_all_roots(_append_rpy_root, gc)
     gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_roots(gc):
-    count = _do_count_rpy_roots(gc)
-    extra = 16
+    # returns a list that may end with some NULLs
     while True:
-        result = [lltype.nullptr(llmemory.GCREF.TO)] * (count + extra)
-        try:
-            _do_append_rpy_roots(gc, result)
-        except ValueError:
-            extra *= 3
-        else:
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * gc._totalroots_rpy
+        count = _do_append_rpy_roots(gc, result)
+        if count <= len(result):     # 'count' fits inside the list
             return result
+        count += (count // 8)
+        gc._totalroots_rpy = count + 10
 
 # ---------- implementation of rpython.rlib.rgc.get_rpy_referents() ----------
 
-def _count_rpy_referent(pointer, gc):
-    gc._count_rpy += 1
-
-def _do_count_rpy_referents(gc, gcref):
-    gc._count_rpy = 0
-    gc.trace(llmemory.cast_ptr_to_adr(gcref), _count_rpy_referent, gc)
-    return gc._count_rpy
-
 def _append_rpy_referent(pointer, gc):
     # Can use the gc list, but should not allocate!
     # It is essential that the list is not resizable!
     lst = gc._list_rpy
     index = gc._count_rpy
-    if index >= len(lst):
-        raise ValueError
     gc._count_rpy = index + 1
-    lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
-                                          llmemory.GCREF)
+    if index < len(lst):
+        lst[index] = llmemory.cast_adr_to_ptr(pointer.address[0],
+                                              llmemory.GCREF)
+    #else:
+    #   too many items.  This situation is detected in the 'while' loop below
 
 def _do_append_rpy_referents(gc, gcref, lst):
     gc._count_rpy = 0
     gc._list_rpy = lst
     gc.trace(llmemory.cast_ptr_to_adr(gcref), _append_rpy_referent, gc)
+    gc._list_rpy = None
+    return gc._count_rpy
 
 def get_rpy_referents(gc, gcref):
-    count = _do_count_rpy_referents(gc, gcref)
-    result = [lltype.nullptr(llmemory.GCREF.TO)] * count
-    _do_append_rpy_referents(gc, gcref, result)
-    return result
+    # returns a list with no NULLs
+    result = []
+    while True:
+        count = _do_append_rpy_referents(gc, gcref, result)
+        if count <= len(result):     # 'count' fits inside the list
+            if count < len(result):
+                result = result[:count]
+            return result
+        result = [lltype.nullptr(llmemory.GCREF.TO)] * count
 
 # ----------
 
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -1399,8 +1399,15 @@
                         if cardbyte & 1:
                             if interval_stop > length:
                                 interval_stop = length
-                                ll_assert(cardbyte <= 1 and bytes == 0,
-                                          "premature end of object")
+                                #--- the sanity check below almost always
+                                #--- passes, except in situations like
+                                #--- test_writebarrier_before_copy_manually\
+                                #    _copy_card_bits
+                                #ll_assert(cardbyte <= 1 and bytes == 0,
+                                #          "premature end of object")
+                                ll_assert(bytes == 0, "premature end of object")
+                                if interval_stop <= interval_start:
+                                    break
                             self.trace_and_drag_out_of_nursery_partial(
                                 obj, interval_start, interval_stop)
                         #
@@ -1780,6 +1787,11 @@
         MovingGCBase.enumerate_all_roots(self, callback, arg)
     enumerate_all_roots._annspecialcase_ = 'specialize:arg(1)'
 
+    def enum_live_with_finalizers(self, callback, arg):
+        self.probably_young_objects_with_finalizers.foreach(callback, arg, 2)
+        self.old_objects_with_finalizers.foreach(callback, arg, 2)
+    enum_live_with_finalizers._annspecialcase_ = 'specialize:arg(1)'
+
     @staticmethod
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
diff --git a/rpython/memory/support.py b/rpython/memory/support.py
--- a/rpython/memory/support.py
+++ b/rpython/memory/support.py
@@ -269,22 +269,23 @@
             self.index_in_oldest = index + 1
             return result
 
-        def foreach(self, callback, arg):
+        def foreach(self, callback, arg, step=1):
             """Invoke 'callback(address, arg)' for all addresses in the deque.
             Typically, 'callback' is a bound method and 'arg' can be None.
+            If step > 1, only calls it for addresses multiple of 'step'.
             """
             chunk = self.oldest_chunk
             index = self.index_in_oldest
             while chunk is not self.newest_chunk:
                 while index < chunk_size:
                     callback(chunk.items[index], arg)
-                    index += 1
+                    index += step
                 chunk = chunk.next
-                index = 0
+                index -= chunk_size
             limit = self.index_in_newest
             while index < limit:
                 callback(chunk.items[index], arg)
-                index += 1
+                index += step
         foreach._annspecialcase_ = 'specialize:arg(1)'
 
         def delete(self):
diff --git a/rpython/memory/test/gc_test_base.py b/rpython/memory/test/gc_test_base.py
--- a/rpython/memory/test/gc_test_base.py
+++ b/rpython/memory/test/gc_test_base.py
@@ -995,6 +995,31 @@
 
         self.interpret(fn, [])
 
+    def test_writebarrier_before_copy_manually_copy_card_bits(self):
+        S = lltype.GcStruct('S', ('x', lltype.Char))
+        TP = lltype.GcArray(lltype.Ptr(S))
+        def fn():
+            l1 = lltype.malloc(TP, 65)
+            l2 = lltype.malloc(TP, 33)
+            for i in range(65):
+                l1[i] = lltype.malloc(S)
+            l = lltype.malloc(TP, 100)
+            i = 0
+            while i < 65:
+                l[i] = l1[i]
+                i += 1
+            rgc.ll_arraycopy(l, l2, 0, 0, 33)
+            x = []
+            # force minor collect
+            t = (1, lltype.malloc(S))
+            for i in range(20):
+                x.append(t)
+            for i in range(33):
+                assert l2[i] == l[i]
+            return 0
+
+        self.interpret(fn, [])
+
     def test_stringbuilder(self):
         def fn():
             s = StringBuilder(4)
diff --git a/rpython/memory/test/test_support.py b/rpython/memory/test/test_support.py
--- a/rpython/memory/test/test_support.py
+++ b/rpython/memory/test/test_support.py
@@ -160,6 +160,10 @@
 
             ll.foreach(callback, 42)
             assert seen == addrs
+            seen = []
+            ll.foreach(callback, 42, step=2)
+            assert seen == addrs[::2]
+
             for a in addrs:
                 b = ll.popleft()
                 assert a == b
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -845,6 +845,9 @@
     @jit.dont_look_inside
     def getsockopt_int(self, level, option):
         flag_p = lltype.malloc(rffi.INTP.TO, 1, flavor='raw')
+        # some win32 calls use only a byte to represent a bool
+        # zero out so the result is correct anyway
+        flag_p[0] = rffi.cast(rffi.INT, 0)
         try:
             flagsize_p = lltype.malloc(_c.socklen_t_ptr.TO, flavor='raw')
             try:
diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -488,19 +488,19 @@
                 deps.append('icon.res')
                 wdeps.append('icon.res')
             m.rule('$(DEFAULT_TARGET)', ['$(TARGET)'] + deps,
-                   ['$(CC_LINK) /nologo /debug %s ' % (' '.join(deps),) + \
+                   ['$(CC_LINK) /nologo /debug /LARGEADDRESSAWARE %s ' % (' '.join(deps),) + \
                     '$(SHARED_IMPORT_LIB) /out:$@ ' + \
                     '/MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
             m.rule('$(WTARGET)', ['$(TARGET)'] + wdeps,
-                   ['$(CC_LINK) /nologo /debug /SUBSYSTEM:WINDOWS %s ' % (' '.join(wdeps),) + \
-                    '$(SHARED_IMPORT_LIB) /out:$@ ' + \
+                   ['$(CC_LINK) /nologo /debug /LARGEADDRESSAWARE /SUBSYSTEM:WINDOWS %s ' % (
+                    ' '.join(wdeps),) + '$(SHARED_IMPORT_LIB) /out:$@ ' + \
                     '/MANIFEST /MANIFESTFILE:$*.manifest',
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
             m.rule('debugmode_$(DEFAULT_TARGET)', ['debugmode_$(TARGET)']+deps,
-                   ['$(CC_LINK) /nologo /DEBUG %s ' % (' '.join(deps),) + \
+                   ['$(CC_LINK) /nologo /DEBUG /LARGEADDRESSAWARE %s ' % (' '.join(deps),) + \
                     'debugmode_$(SHARED_IMPORT_LIB) /out:$@',
                     ])
 

From pypy.commits at gmail.com  Fri Sep 15 12:42:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 15 Sep 2017 09:42:57 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the tests for pycparser 2.18
Message-ID: <59bc0311.c55c1c0a.1b420.e4b2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3009:b19be645c9f5
Date: 2017-09-15 18:42 +0200
http://bitbucket.org/cffi/cffi/changeset/b19be645c9f5/

Log:	Update the tests for pycparser 2.18

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -1,6 +1,6 @@
 
 import sys, os, py
-from cffi import FFI, VerificationError, FFIError
+from cffi import FFI, VerificationError, FFIError, CDefError
 from cffi import recompiler
 from testing.udir import udir
 from testing.support import u, long
@@ -1126,7 +1126,9 @@
 
 def test_some_float_invalid_1():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, "typedef long double... foo_t;")
 
 def test_some_float_invalid_2():
     ffi = FFI()
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -1,5 +1,6 @@
 import os, sys, math, py
 from cffi import FFI, FFIError, VerificationError, VerificationMissing, model
+from cffi import CDefError
 from cffi import recompiler
 from testing.support import *
 import _cffi_backend
@@ -2222,7 +2223,9 @@
 
 def test_unsupported_some_primitive_types():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, """typedef void... foo_t;""")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, """typedef void... foo_t;""")
     #
     ffi.cdef("typedef int... foo_t;")
     py.test.raises(VerificationError, ffi.verify, "typedef float foo_t;")

From pypy.commits at gmail.com  Fri Sep 15 12:43:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 15 Sep 2017 09:43:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/b19be645c9f5
Message-ID: <59bc033e.098b1c0a.da132.0a68@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92405:566bf0c6ad4d
Date: 2017-09-15 18:43 +0200
http://bitbucket.org/pypy/pypy/changeset/566bf0c6ad4d/

Log:	update to cffi/b19be645c9f5

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1,7 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 
 import sys, os, py
-from cffi import FFI, VerificationError, FFIError
+from cffi import FFI, VerificationError, FFIError, CDefError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import u, long
@@ -1127,7 +1127,9 @@
 
 def test_some_float_invalid_1():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, "typedef long double... foo_t;")
 
 def test_some_float_invalid_2():
     ffi = FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -1,6 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 import os, sys, math, py
 from cffi import FFI, FFIError, VerificationError, VerificationMissing, model
+from cffi import CDefError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.support import *
 import _cffi_backend
@@ -2223,7 +2224,9 @@
 
 def test_unsupported_some_primitive_types():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, """typedef void... foo_t;""")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, """typedef void... foo_t;""")
     #
     ffi.cdef("typedef int... foo_t;")
     py.test.raises(VerificationError, ffi.verify, "typedef float foo_t;")

From pypy.commits at gmail.com  Fri Sep 15 13:24:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 15 Sep 2017 10:24:56 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update the doc,
 as suggested on python-cffi
Message-ID: <59bc0ce8.06aadf0a.e1044.5cfe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3010:62beffad167e
Date: 2017-09-15 19:24 +0200
http://bitbucket.org/cffi/cffi/changeset/62beffad167e/

Log:	Update the doc, as suggested on python-cffi

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -128,10 +128,37 @@
 
 There is no general equivalent to the ``&`` operator in C (because it
 would not fit nicely in the model, and it does not seem to be needed
-here).  But see `ffi.addressof()`__.
+here).  There is `ffi.addressof()`__, but only for some cases.  You
+cannot take the "address" of a number in Python, for example; similarly,
+you cannot take the address of a CFFI pointer.  If you have this kind
+of C code::
+
+    int x, y;
+    fetch_size(&x, &y);
+
+    opaque_t *handle;      // some opaque pointer
+    init_stuff(&handle);   // initializes the variable 'handle'
+    more_stuff(handle);    // pass the handle around to more functions
+
+then you need to rewrite it like this, replacing the variables in C
+with what is logically pointers to the variables:
+
+.. code-block:: python
+
+    px = ffi.new("int *")
+    py = ffi.new("int *")              arr = ffi.new("int[2]")
+    lib.fetch_size(px, py)    -OR-     lib.fetch_size(arr, arr + 1)
+    x = px[0]                          x = arr[0]
+    y = py[0]                          y = arr[1]
+
+    p_handle = ffi.new("opaque_t **")
+    lib.init_stuff(p_handle)   # pass the pointer to the 'handle' pointer
+    handle = p_handle[0]       # now we can read 'handle' out of 'p_handle'
+    lib.more_stuff(handle)
 
 .. __: ref.html#ffi-addressof
 
+
 Any operation that would in C return a pointer or array or struct type
 gives you a fresh cdata object.  Unlike the "original" one, these fresh
 cdata objects don't have ownership: they are merely references to
@@ -208,7 +235,7 @@
 string stored in the source array (adding surrogates if necessary).
 See the `Unicode character types`__ section for more details.
 
-__: ref.html#unichar
+.. __: ref.html#unichar
 
 Note that unlike Python lists or tuples, but like C, you *cannot* index in
 a C array from the end using negative numbers.

From pypy.commits at gmail.com  Fri Sep 15 16:58:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 13:58:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add inefficient implementation of
 PyUnicode_FromKindAndData()
Message-ID: <59bc3eff.6588df0a.943d7.0b5e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92406:e27c61e1a09a
Date: 2017-09-15 21:58 +0100
http://bitbucket.org/pypy/pypy/changeset/e27c61e1a09a/

Log:	Add inefficient implementation of PyUnicode_FromKindAndData()

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -154,6 +154,42 @@
         res = module.test_unicode_format(1, "xyz")
         assert res == "bla 1 ble xyz\n"
 
+    def test_fromkind(self):
+        module = self.import_extension('foo', [
+            ('from_ucs1', 'METH_O',
+             """
+             char* p;
+             Py_ssize_t size;
+             if (PyBytes_AsStringAndSize(args, &p, &size) < 0)
+                return NULL;
+             return PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, p, size);
+             """),
+            ('from_ucs2', 'METH_O',
+             """
+             char* p;
+             Py_ssize_t size;
+             if (PyBytes_AsStringAndSize(args, &p, &size) < 0)
+                return NULL;
+             return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, p, size/2);
+             """),
+            ('from_ucs4', 'METH_O',
+             """
+             char* p;
+             Py_ssize_t size;
+             if (PyBytes_AsStringAndSize(args, &p, &size) < 0)
+                return NULL;
+             return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, p, size/4);
+             """)])
+        res = module.from_ucs1(b'spam')
+        assert res == 'spam'
+        s = "sp&#228;m"
+        b = s.encode('utf-16')[2:]  # Skip the BOM
+        s2 = module.from_ucs2(b)
+        assert module.from_ucs2(b) == s
+        s = "x\N{PILE OF POO}x"
+        b = s.encode('utf-32')[4:]  # Skip the BOM
+        assert module.from_ucs4(b) == s
+
     def test_aswidecharstring(self):
         module = self.import_extension('foo', [
             ("aswidecharstring", "METH_O",
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -13,7 +13,8 @@
     PyObject, PyObjectP, Py_DecRef, make_ref, from_ref, track_reference,
     make_typedescr, get_typedescr, as_pyobj)
 from pypy.module.cpyext.bytesobject import PyBytes_Check, PyBytes_FromObject
-from pypy.module._codecs.interp_codecs import CodecState
+from pypy.module._codecs.interp_codecs import (
+    CodecState, latin_1_decode, utf_16_decode, utf_32_decode)
 from pypy.objspace.std import unicodeobject
 from rpython.rlib import rstring, runicode
 from rpython.tool.sourcetools import func_renamer
@@ -34,7 +35,7 @@
                    dealloc=unicode_dealloc,
                    realize=unicode_realize)
 
-# Buffer for the default encoding (used by PyUnicde_GetDefaultEncoding)
+# Buffer for the default encoding (used by PyUnicode_GetDefaultEncoding)
 DEFAULT_ENCODING_SIZE = 100
 default_encoding = lltype.malloc(rffi.CCHARP.TO, DEFAULT_ENCODING_SIZE,
                                  flavor='raw', zero=True)
@@ -307,6 +308,26 @@
     set_ready(py_obj, 1)
     return 0
 
+ at cts.decl("""PyObject* PyUnicode_FromKindAndData(
+        int kind, const void *buffer, Py_ssize_t size)""")
+def PyUnicode_FromKindAndData(space, kind, data, size):
+    if size < 0:
+        raise oefmt(space.w_ValueError, "size must be positive")
+    if kind == _1BYTE_KIND:
+        value = rffi.charpsize2str(data, size)
+        w_res = latin_1_decode(space, value, w_final=space.w_False)
+    elif kind == _2BYTE_KIND:
+        value = rffi.charpsize2str(data, 2 * size)
+        w_res = utf_16_decode(space, value, w_final=space.w_False)
+    elif kind == _4BYTE_KIND:
+        value = rffi.charpsize2str(data, 4 * size)
+        w_res = utf_32_decode(space, value, w_final=space.w_False)
+    else:
+        raise oefmt(space.w_SystemError, "invalid kind")
+    w_ret = space.unpackiterable(w_res)[0]
+    _PyUnicode_Ready(space, w_ret)
+    return w_ret
+
 @cts.decl("Py_UNICODE * PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size)")
 def PyUnicode_AsUnicodeAndSize(space, ref, psize):
     """Return a read-only pointer to the Unicode object's internal Py_UNICODE

From pypy.commits at gmail.com  Fri Sep 15 17:33:58 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 14:33:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <59bc4746.952d1c0a.1f219.de24@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92407:5ad69386dfac
Date: 2017-09-15 22:33 +0100
http://bitbucket.org/pypy/pypy/changeset/5ad69386dfac/

Log:	fix translation

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -313,6 +313,7 @@
 def PyUnicode_FromKindAndData(space, kind, data, size):
     if size < 0:
         raise oefmt(space.w_ValueError, "size must be positive")
+    data = cts.cast('char *', data)
     if kind == _1BYTE_KIND:
         value = rffi.charpsize2str(data, size)
         w_res = latin_1_decode(space, value, w_final=space.w_False)

From pypy.commits at gmail.com  Fri Sep 15 17:56:00 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 14:56:00 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation again
Message-ID: <59bc4c70.e186df0a.ac189.b1fa@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92408:a874c797101f
Date: 2017-09-15 22:55 +0100
http://bitbucket.org/pypy/pypy/changeset/a874c797101f/

Log:	fix translation again

diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -314,6 +314,7 @@
     if size < 0:
         raise oefmt(space.w_ValueError, "size must be positive")
     data = cts.cast('char *', data)
+    kind = widen(kind)
     if kind == _1BYTE_KIND:
         value = rffi.charpsize2str(data, size)
         w_res = latin_1_decode(space, value, w_final=space.w_False)

From pypy.commits at gmail.com  Sat Sep 16 00:16:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 15 Sep 2017 21:16:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement PyDict_SetDefault()
Message-ID: <59bca58b.0582df0a.a217c.fa33@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92409:1a05dfedf02e
Date: 2017-09-16 05:15 +0100
http://bitbucket.org/pypy/pypy/changeset/1a05dfedf02e/

Log:	Implement PyDict_SetDefault()

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -4,10 +4,10 @@
 from pypy.objspace.std.classdict import ClassDictStrategy
 from pypy.interpreter.typedef import GetSetProperty
 from pypy.module.cpyext.api import (
-    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t,
+    cpython_api, CANNOT_FAIL, build_type_checkers_flags, Py_ssize_t, cts,
     Py_ssize_tP, CONST_STRING, PyObjectFields, cpython_struct,
     bootstrap_function, slot_function)
-from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj, 
+from pypy.module.cpyext.pyobject import (PyObject, PyObjectP, as_pyobj,
         make_typedescr, track_reference, create_ref, from_ref, decref,
         Py_IncRef)
 from pypy.module.cpyext.object import _dealloc
@@ -155,6 +155,15 @@
     """Empty an existing dictionary of all key-value pairs."""
     space.call_method(space.w_dict, "clear", w_obj)
 
+ at cts.decl("""PyObject *
+    PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *defaultobj)""")
+def PyDict_SetDefault(space, w_dict, w_key, w_defaultobj):
+    if not PyDict_Check(space, w_dict):
+        PyErr_BadInternalCall(space)
+    else:
+        return space.call_method(
+            space.w_dict, "setdefault", w_dict, w_key, w_defaultobj)
+
 @cpython_api([PyObject], PyObject)
 def PyDict_Copy(space, w_obj):
     """Return a new dictionary that contains the same key-value pairs as p.
@@ -258,7 +267,7 @@
     if w_dict is None:
         return 0
     if not space.isinstance_w(w_dict, space.w_dict):
-        return 0 
+        return 0
     pos = ppos[0]
     py_obj = as_pyobj(space, w_dict)
     py_dict = rffi.cast(PyDictObject, py_obj)
diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py
--- a/pypy/module/cpyext/test/test_dictobject.py
+++ b/pypy/module/cpyext/test/test_dictobject.py
@@ -174,6 +174,26 @@
             ])
         assert module.dict_proxy({'a': 1, 'b': 2}) == 2
 
+    def test_setdefault(self):
+        module = self.import_extension('foo', [
+            ("setdefault", "METH_VARARGS",
+             '''
+             PyObject *d, *key, *defaultobj, *val;
+             if (!PyArg_ParseTuple(args, "OOO", &d, &key, &defaultobj))
+                 return NULL;
+             val = PyDict_SetDefault(d, key, defaultobj);
+             Py_XINCREF(val);
+             return val;
+             ''')])
+
+        class Dict(dict):
+            def setdefault(self, key, default):
+                return 42
+
+        d = Dict()
+        assert module.setdefault(d, 'x', 1) == 1
+        assert d['x'] == 1
+
     def test_update(self):
         module = self.import_extension('foo', [
             ("update", "METH_VARARGS",

From pypy.commits at gmail.com  Sat Sep 16 09:13:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 06:13:49 -0700 (PDT)
Subject: [pypy-commit] cffi default: Typo
Message-ID: <59bd238d.86b71c0a.95c08.60dc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3011:900dc93b3c6a
Date: 2017-09-16 15:13 +0200
http://bitbucket.org/cffi/cffi/changeset/900dc93b3c6a/

Log:	Typo

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -11,7 +11,7 @@
   when used as ``charN_t *`` or ``charN_t[]`` they represent a unicode
   string.  The difference with ``wchar_t`` is that they have a known,
   fixed size.  They should work at all places that used to work with
-  ``wchar_t`` (please report an issue if I missing something).  Note
+  ``wchar_t`` (please report an issue if I missed something).  Note
   that with ``set_source()``, you need to make sure that these types are
   actually defined by the C source you provide (if used in ``cdef()``).
 

From pypy.commits at gmail.com  Sat Sep 16 11:38:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:38:24 -0700 (PDT)
Subject: [pypy-commit] cffi default: merge heads
Message-ID: <59bd4570.0f141c0a.5bfd0.71bb@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3013:7417b70e5b1a
Date: 2017-09-16 15:43 +0200
http://bitbucket.org/cffi/cffi/changeset/7417b70e5b1a/

Log:	merge heads

diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -128,10 +128,37 @@
 
 There is no general equivalent to the ``&`` operator in C (because it
 would not fit nicely in the model, and it does not seem to be needed
-here).  But see `ffi.addressof()`__.
+here).  There is `ffi.addressof()`__, but only for some cases.  You
+cannot take the "address" of a number in Python, for example; similarly,
+you cannot take the address of a CFFI pointer.  If you have this kind
+of C code::
+
+    int x, y;
+    fetch_size(&x, &y);
+
+    opaque_t *handle;      // some opaque pointer
+    init_stuff(&handle);   // initializes the variable 'handle'
+    more_stuff(handle);    // pass the handle around to more functions
+
+then you need to rewrite it like this, replacing the variables in C
+with what is logically pointers to the variables:
+
+.. code-block:: python
+
+    px = ffi.new("int *")
+    py = ffi.new("int *")              arr = ffi.new("int[2]")
+    lib.fetch_size(px, py)    -OR-     lib.fetch_size(arr, arr + 1)
+    x = px[0]                          x = arr[0]
+    y = py[0]                          y = arr[1]
+
+    p_handle = ffi.new("opaque_t **")
+    lib.init_stuff(p_handle)   # pass the pointer to the 'handle' pointer
+    handle = p_handle[0]       # now we can read 'handle' out of 'p_handle'
+    lib.more_stuff(handle)
 
 .. __: ref.html#ffi-addressof
 
+
 Any operation that would in C return a pointer or array or struct type
 gives you a fresh cdata object.  Unlike the "original" one, these fresh
 cdata objects don't have ownership: they are merely references to
@@ -208,7 +235,7 @@
 string stored in the source array (adding surrogates if necessary).
 See the `Unicode character types`__ section for more details.
 
-__: ref.html#unichar
+.. __: ref.html#unichar
 
 Note that unlike Python lists or tuples, but like C, you *cannot* index in
 a C array from the end using negative numbers.
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -11,7 +11,7 @@
   when used as ``charN_t *`` or ``charN_t[]`` they represent a unicode
   string.  The difference with ``wchar_t`` is that they have a known,
   fixed size.  They should work at all places that used to work with
-  ``wchar_t`` (please report an issue if I missing something).  Note
+  ``wchar_t`` (please report an issue if I missed something).  Note
   that with ``set_source()``, you need to make sure that these types are
   actually defined by the C source you provide (if used in ``cdef()``).
 

From pypy.commits at gmail.com  Sat Sep 16 11:38:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:38:22 -0700 (PDT)
Subject: [pypy-commit] cffi default: Python 3
Message-ID: <59bd456e.88b0df0a.f8abf.5a4f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3012:10ab414ad7b2
Date: 2017-09-16 15:43 +0200
http://bitbucket.org/cffi/cffi/changeset/10ab414ad7b2/

Log:	Python 3

diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2466,8 +2466,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
 
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2303,8 +2303,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
         # with PyPy's GC, the above would rapidly consume 40 GB of RAM

From pypy.commits at gmail.com  Sat Sep 16 11:44:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:44:54 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.11: md5/sha and link fix(?)
Message-ID: <59bd46f6.56811c0a.bec63.4bf5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.11
Changeset: r3015:296f96edb561
Date: 2017-09-16 17:43 +0200
http://bitbucket.org/cffi/cffi/changeset/296f96edb561/

Log:	md5/sha and link fix(?)

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.11.0.tar.gz
+* https://pypi.python.org/packages/4e/32/4070bdf32812c89eb635c80880a5caa2e0189aa7999994c265577e5154f3/cffi-1.11.0.tar.gz#md5=2c5939cc2fa0183fe0c2fcb6a4f1994d
 
-   - MD5: ...
+   - MD5: 2c5939cc2fa0183fe0c2fcb6a4f1994d
 
-   - SHA: ...
+   - SHA: 93cb5aaf152e19f9d4082a723aa2396e9cd5d93f
 
-   - SHA256: ...
+   - SHA256: 5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Sep 16 11:44:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:44:52 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.11: release branch
Message-ID: <59bd46f4.4fabdf0a.e6bbd.1c65@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.11
Changeset: r3014:f18bef49a17c
Date: 2017-09-16 17:39 +0200
http://bitbucket.org/cffi/cffi/changeset/f18bef49a17c/

Log:	release branch


From pypy.commits at gmail.com  Sat Sep 16 11:44:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:44:56 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge release-1.11
Message-ID: <59bd46f8.0ec61c0a.be519.b3c3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3016:451b524ea386
Date: 2017-09-16 17:44 +0200
http://bitbucket.org/cffi/cffi/changeset/451b524ea386/

Log:	hg merge release-1.11

diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.11.0.tar.gz
+* https://pypi.python.org/packages/4e/32/4070bdf32812c89eb635c80880a5caa2e0189aa7999994c265577e5154f3/cffi-1.11.0.tar.gz#md5=2c5939cc2fa0183fe0c2fcb6a4f1994d
 
-   - MD5: ...
+   - MD5: 2c5939cc2fa0183fe0c2fcb6a4f1994d
 
-   - SHA: ...
+   - SHA: 93cb5aaf152e19f9d4082a723aa2396e9cd5d93f
 
-   - SHA256: ...
+   - SHA256: 5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``

From pypy.commits at gmail.com  Sat Sep 16 11:48:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 08:48:20 -0700 (PDT)
Subject: [pypy-commit] cffi default: Generalize the test to pass
Message-ID: <59bd47c4.84901c0a.5ee6a.c846@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3017:6586a1f5720d
Date: 2017-09-16 17:48 +0200
http://bitbucket.org/cffi/cffi/changeset/6586a1f5720d/

Log:	Generalize the test to pass

diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py
--- a/testing/cffi0/test_version.py
+++ b/testing/cffi0/test_version.py
@@ -36,7 +36,7 @@
     v = cffi.__version__.replace('+', '')
     p = os.path.join(parent, 'doc', 'source', 'installation.rst')
     content = open(p).read()
-    assert ("cffi/cffi-%s.tar.gz" % v) in content
+    assert ("/cffi-%s.tar.gz" % v) in content
 
 def test_setup_version():
     parent = os.path.dirname(os.path.dirname(cffi.__file__))

From pypy.commits at gmail.com  Sat Sep 16 13:29:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 16 Sep 2017 10:29:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix 2BYTE case in _PyUnicode_Ready():
 don't prepend a BOM to the data
Message-ID: <59bd5f65.84e51c0a.d0a72.85d7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92410:b6ba2262940e
Date: 2017-09-16 18:28 +0100
http://bitbucket.org/pypy/pypy/changeset/b6ba2262940e/

Log:	Fix 2BYTE case in _PyUnicode_Ready(): don't prepend a BOM to the
	data

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -190,6 +190,26 @@
         b = s.encode('utf-32')[4:]  # Skip the BOM
         assert module.from_ucs4(b) == s
 
+    def test_substring(self):
+        module = self.import_extension('foo', [
+            ("slice_start", "METH_VARARGS",
+             '''
+             PyObject* text;
+             Py_ssize_t start, length;
+             if (!PyArg_ParseTuple(args, "On", &text, &start))
+                return NULL;
+             if (PyUnicode_READY(text) == -1) return NULL;
+             length = PyUnicode_GET_LENGTH(text);
+             if (start > length) return PyLong_FromSsize_t(start);
+             return PyUnicode_FromKindAndData(PyUnicode_KIND(text),
+                 PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text),
+                 length-start);
+             ''')])
+        s = 'a&#1040;b&#1041;c&#1057;d&#1044;'
+        assert module.slice_start(s, 2) == 'b&#1041;c&#1057;d&#1044;'
+        s = 'xx\N{PILE OF POO}'
+        assert module.slice_start(s, 2) == '\N{PILE OF POO}'
+
     def test_aswidecharstring(self):
         module = self.import_extension('foo', [
             ("aswidecharstring", "METH_O",
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1,6 +1,6 @@
 from pypy.interpreter.error import OperationError, oefmt
 from rpython.rtyper.lltypesystem import rffi, lltype
-from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16
+from rpython.rlib.runicode import unicode_encode_latin_1, unicode_encode_utf_16_helper
 from rpython.rlib.rarithmetic import widen
 
 from pypy.module.unicodedata import unicodedb
@@ -289,8 +289,9 @@
             set_utf8_len(py_obj, 0)
     elif maxchar < 65536:
         # XXX: assumes that sizeof(wchar_t) == 4
-        ucs2_str = unicode_encode_utf_16(
-            w_obj._value, len(w_obj._value), errors='strict')
+        ucs2_str = unicode_encode_utf_16_helper(
+            w_obj._value, len(w_obj._value), errors='strict',
+            byteorder=runicode.BYTEORDER)
         ucs2_data = cts.cast('Py_UCS2 *', rffi.str2charp(ucs2_str))
         set_data(py_obj, cts.cast('void*', ucs2_data))
         set_len(py_obj, get_wsize(py_obj))

From pypy.commits at gmail.com  Sat Sep 16 13:29:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 10:29:30 -0700 (PDT)
Subject: [pypy-commit] buildbot default: fix
Message-ID: <59bd5f7a.09a0df0a.6c233.e4f5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r1029:7c00a8e159c3
Date: 2017-09-16 19:29 +0200
http://bitbucket.org/pypy/buildbot/changeset/7c00a8e159c3/

Log:	fix

diff --git a/master/templates/layout.html b/master/templates/layout.html
--- a/master/templates/layout.html
+++ b/master/templates/layout.html
@@ -27,7 +27,7 @@
         - <a href="http://speed.pypy.org/">Speed</a>
         - <!-- a href="{{ path_to_root }}numpy-status/">NumPy compatibility</a -->
         - <a href="{{ path_to_root }}summary?branch=&lt;trunk&gt;">Summary (trunk)</a>
-        - <a href="{{ path_to_root }}summary?branch=3.5">Summary (py3.5)</a>
+        - <a href="{{ path_to_root }}summary?branch=py3.5">Summary (py3.5)</a>
         - <a href="{{ path_to_root }}summary">Summary</a>
         - <a href="{{ path_to_root }}nightly/">Nightly builds</a>
         <!-- end of PyPy specific items -->

From pypy.commits at gmail.com  Sat Sep 16 15:24:06 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Sat, 16 Sep 2017 12:24:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: handle SO_REUSEPORT defined but not
 implemented
Message-ID: <59bd7a56.43001c0a.92aa1.8183@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3.5
Changeset: r92411:8086c7a846ff
Date: 2017-09-16 12:19 -0700
http://bitbucket.org/pypy/pypy/changeset/8086c7a846ff/

Log:	handle SO_REUSEPORT defined but not implemented

	follows up 7ade09ee3ac5, see also https://bugs.python.org/issue26858

diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -1588,9 +1588,15 @@
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEADDR))
         if reuseport_supported:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            try:
+                self.assertFalse(
+                    sock.getsockopt(
+                        socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            except OSError:
+                # Python's socket module was compiled using modern headers
+                # thus defining SO_REUSEPORT but this process is running
+                # under an older kernel that does not support SO_REUSEPORT.
+                reuseport_supported = False
         self.assertFalse(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))

From pypy.commits at gmail.com  Sun Sep 17 02:38:52 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 23:38:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2658
Message-ID: <59be187c.a8a6df0a.3cb62.b87a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92412:5a5707c54da3
Date: 2017-09-17 08:38 +0200
http://bitbucket.org/pypy/pypy/changeset/5a5707c54da3/

Log:	Issue #2658

	Comment out these Windows asserts. It's fragile even on top of
	CPython, and seems not to work at all on top of PyPy.

diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -109,6 +109,7 @@
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
 def GetLastError(space):
+    """NOTE: don't use this.  See issue #2658"""
     return space.newint(rwin32.GetLastError_saved())
 
 # __________________________________________________________

From pypy.commits at gmail.com  Sun Sep 17 02:45:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 16 Sep 2017 23:45:36 -0700 (PDT)
Subject: [pypy-commit] cffi default: Add Python 3.6 in that list
Message-ID: <59be1a10.7db9df0a.a1c52.e980@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3018:f61db8830105
Date: 2017-09-17 08:45 +0200
http://bitbucket.org/cffi/cffi/changeset/f61db8830105/

Log:	Add Python 3.6 in that list

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -231,6 +231,7 @@
             'Programming Language :: Python :: 3.3',
             'Programming Language :: Python :: 3.4',
             'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
             'Programming Language :: Python :: Implementation :: CPython',
             'Programming Language :: Python :: Implementation :: PyPy',
         ],

From pypy.commits at gmail.com  Mon Sep 18 13:08:11 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 18 Sep 2017 10:08:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: update checksum for newer local_59.zip
 third party package
Message-ID: <59bffd7b.c2311c0a.deab2.ef89@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92413:2a906be41e03
Date: 2017-09-18 20:06 +0300
http://bitbucket.org/pypy/pypy/changeset/2a906be41e03/

Log:	update checksum for newer local_59.zip third party package

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_59.zip
 (for post-5.8 builds) with sha256 checksum
-``0f96c045db1f5f73ad0fae7857caa69c261324bd8e51f6d2ad1fa842c4a5f26f``
+``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or

From pypy.commits at gmail.com  Mon Sep 18 13:42:09 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 18 Sep 2017 10:42:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: revert part of 921cf6b61598,
 assert for translation
Message-ID: <59c00571.84891c0a.85f5d.741e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92414:d5cf625c28e3
Date: 2017-09-18 20:41 +0300
http://bitbucket.org/pypy/pypy/changeset/d5cf625c28e3/

Log:	revert part of 921cf6b61598, assert for translation

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -109,21 +109,27 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
+        w_objclass = self.w_objclass 
+        assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
-            self.name, self.w_objclass.getname(self.space)))
+            self.name, w_objclass.name))
 
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' of '%s' object needs an argument",
-                self.name, self.w_objclass.getname(self.space))
+                self.name, w_objclass.name)
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' requires a '%s' object but received a '%T'",
-                self.name, self.w_objclass.getname(self.space), w_instance)
+                self.name, w_objclass.name, w_instance)
         w_args = space.newtuple(args_w[1:])
         w_kw = space.newdict()
         for key, w_obj in kw_w.items():

From pypy.commits at gmail.com  Mon Sep 18 13:52:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 18 Sep 2017 10:52:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: {sg}etdlopenflags not available on
 win32, makes cffi test failure into a skip
Message-ID: <59c007db.499edf0a.d5eeb.41b2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92415:204b542686ae
Date: 2017-09-18 20:51 +0300
http://bitbucket.org/pypy/pypy/changeset/204b542686ae/

Log:	{sg}etdlopenflags not available on win32, makes cffi test failure
	into a skip

diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -91,13 +91,14 @@
         'float_info'            : 'system.get_float_info(space)',
         'long_info'             : 'system.get_long_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
-        'getdlopenflags'        : 'system.getdlopenflags',
-        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
         interpleveldefs['winver'] = 'version.get_winver(space)'
         interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion'
+    else:
+        interpleveldefs['getdlopenflags'] = 'system.getdlopenflags'
+        interpleveldefs['setdlopenflags'] = 'system.setdlopenflags'
 
     appleveldefs = {
         'excepthook'            : 'app.excepthook',

From pypy.commits at gmail.com  Mon Sep 18 14:09:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 18 Sep 2017 11:09:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Correctly initialize data in 4BYTE case
 of _PyUnicode_Ready(),
 and fix test to behave the same whether translated or not
Message-ID: <59c00bc9.196b1c0a.33197.83a9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92416:ffe904e9ac64
Date: 2017-09-18 19:08 +0100
http://bitbucket.org/pypy/pypy/changeset/ffe904e9ac64/

Log:	Correctly initialize data in 4BYTE case of _PyUnicode_Ready(), and
	fix test to behave the same whether translated or not

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -199,16 +199,21 @@
              if (!PyArg_ParseTuple(args, "On", &text, &start))
                 return NULL;
              if (PyUnicode_READY(text) == -1) return NULL;
+             if (!PyUnicode_1BYTE_DATA(text)) {
+                // Don't segfault, just fail the test.
+                Py_RETURN_NONE;
+             }
              length = PyUnicode_GET_LENGTH(text);
              if (start > length) return PyLong_FromSsize_t(start);
              return PyUnicode_FromKindAndData(PyUnicode_KIND(text),
                  PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text),
                  length-start);
              ''')])
-        s = 'a&#1040;b&#1041;c&#1057;d&#1044;'
+        s = u'a&#1040;b&#1041;c&#1057;d&#1044;'
         assert module.slice_start(s, 2) == 'b&#1041;c&#1057;d&#1044;'
-        s = 'xx\N{PILE OF POO}'
-        assert module.slice_start(s, 2) == '\N{PILE OF POO}'
+        # s = u'xx\N{PILE OF POO}'
+        s = u'xx\U0001F4A9'
+        assert module.slice_start(s, 2) == u'\U0001F4A9'
 
     def test_aswidecharstring(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -300,6 +300,11 @@
         set_utf8_len(py_obj, 0)
     else:
         # XXX: assumes that sizeof(wchar_t) == 4
+        if not get_wbuffer(py_obj):
+            # Copy unicode buffer
+            u = w_obj._value
+            set_wbuffer(py_obj, rffi.unicode2wcharp(u))
+            set_wsize(py_obj, len(u))
         ucs4_data = get_wbuffer(py_obj)
         set_data(py_obj, cts.cast('void*', ucs4_data))
         set_len(py_obj, get_wsize(py_obj))

From pypy.commits at gmail.com  Tue Sep 19 06:14:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 19 Sep 2017 03:14:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2659
Message-ID: <59c0edfb.7a86df0a.cd3c4.504d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92417:435111355ce7
Date: 2017-09-19 11:52 +0200
http://bitbucket.org/pypy/pypy/changeset/435111355ce7/

Log:	Issue #2659

	Test and fix for a special case

diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -419,3 +419,7 @@
         def f():
             return x
         assert f.__closure__[0].cell_contents is x
+
+    def test_get_with_none_arg(self):
+        raises(TypeError, type.__dict__['__mro__'].__get__, None)
+        raises(TypeError, type.__dict__['__mro__'].__get__, None, None)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -297,6 +297,8 @@
         if (space.is_w(w_obj, space.w_None)
             and not space.is_w(w_cls, space.type(space.w_None))):
             #print self, w_obj, w_cls
+            if space.is_w(w_cls, space.w_None):
+                raise oefmt(space.w_TypeError, "__get__(None, None) is invalid")
             return self
         else:
             try:

From pypy.commits at gmail.com  Tue Sep 19 06:14:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 19 Sep 2017 03:14:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <59c0edfd.51421c0a.8b316.99fa@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92418:a9eafbdf3c28
Date: 2017-09-19 12:13 +0200
http://bitbucket.org/pypy/pypy/changeset/a9eafbdf3c28/

Log:	merge heads

diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_59.zip
 (for post-5.8 builds) with sha256 checksum
-``0f96c045db1f5f73ad0fae7857caa69c261324bd8e51f6d2ad1fa842c4a5f26f``
+``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -109,21 +109,27 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
+        w_objclass = self.w_objclass 
+        assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
-            self.name, self.w_objclass.getname(self.space)))
+            self.name, w_objclass.name))
 
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' of '%s' object needs an argument",
-                self.name, self.w_objclass.getname(self.space))
+                self.name, w_objclass.name)
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' requires a '%s' object but received a '%T'",
-                self.name, self.w_objclass.getname(self.space), w_instance)
+                self.name, w_objclass.name, w_instance)
         w_args = space.newtuple(args_w[1:])
         w_kw = space.newdict()
         for key, w_obj in kw_w.items():
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -91,13 +91,14 @@
         'float_info'            : 'system.get_float_info(space)',
         'long_info'             : 'system.get_long_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
-        'getdlopenflags'        : 'system.getdlopenflags',
-        'setdlopenflags'        : 'system.setdlopenflags',
         }
 
     if sys.platform == 'win32':
         interpleveldefs['winver'] = 'version.get_winver(space)'
         interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion'
+    else:
+        interpleveldefs['getdlopenflags'] = 'system.getdlopenflags'
+        interpleveldefs['setdlopenflags'] = 'system.setdlopenflags'
 
     appleveldefs = {
         'excepthook'            : 'app.excepthook',

From pypy.commits at gmail.com  Tue Sep 19 10:57:20 2017
From: pypy.commits at gmail.com (mattip)
Date: Tue, 19 Sep 2017 07:57:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: on win32,
 sys does not have {gs}etdlopenflags
Message-ID: <59c13050.928e1c0a.7e26a.ba54@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92419:e9e0fd719251
Date: 2017-09-19 17:55 +0300
http://bitbucket.org/pypy/pypy/changeset/e9e0fd719251/

Log:	on win32, sys does not have {gs}etdlopenflags

diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -445,6 +445,8 @@
 
     def test_dlopenflags(self):
         import sys
+        if not hasattr(sys, "getdlopenflags"):
+            skip('{gs}etdlopenflags is not implemented on this platform')
         raises(TypeError, sys.getdlopenflags, 42)
         oldflags = sys.getdlopenflags()
         raises(TypeError, sys.setdlopenflags)

From pypy.commits at gmail.com  Tue Sep 19 15:30:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 19 Sep 2017 12:30:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: A failing Cython-related test
Message-ID: <59c17050.4f541c0a.3bdda.0688@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92420:32f2c8e9403e
Date: 2017-09-19 20:27 +0100
http://bitbucket.org/pypy/pypy/changeset/32f2c8e9403e/

Log:	A failing Cython-related test

diff --git a/pypy/module/cpyext/test/test_exception.py b/pypy/module/cpyext/test/test_exception.py
--- a/pypy/module/cpyext/test/test_exception.py
+++ b/pypy/module/cpyext/test/test_exception.py
@@ -45,3 +45,22 @@
                                      PyExc_IOError);
              """)])
         assert module.get_aliases() == (OSError, OSError)
+
+    def test_implicit_chaining(self):
+        module = self.import_extension('foo', [
+            ("raise_exc", "METH_NOARGS",
+             """
+                PyObject *ev, *et, *tb;
+                PyErr_SetString(PyExc_ValueError, "foo");
+
+                // simplified copy of __Pyx_GetException
+                PyErr_Fetch(&et, &ev, &tb);
+                PyErr_NormalizeException(&et, &ev, &tb);
+                if (tb) PyException_SetTraceback(ev, tb);
+                PyErr_SetExcInfo(et, ev, tb);
+
+                PyErr_SetString(PyExc_TypeError, "bar");
+                return NULL;
+             """)])
+        excinfo = raises(TypeError, module.raise_exc)
+        assert excinfo.value.__context__

From pypy.commits at gmail.com  Wed Sep 20 02:31:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 19 Sep 2017 23:31:57 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2660
Message-ID: <59c20b5d.cd5e1c0a.8263a.563c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92421:79230ba1326e
Date: 2017-09-20 08:31 +0200
http://bitbucket.org/pypy/pypy/changeset/79230ba1326e/

Log:	Issue #2660

	sysconfig.get_config_var('SHLIB_SUFFIX') should just return '.so',
	on Posix systems

diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -73,7 +73,7 @@
     g['CCSHARED'] = "-fPIC"
     g['LDSHARED'] = "cc -pthread -shared"
     g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = so_ext
+    g['SHLIB_SUFFIX'] = ".so"
     g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"

From pypy.commits at gmail.com  Wed Sep 20 02:55:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 19 Sep 2017 23:55:22 -0700 (PDT)
Subject: [pypy-commit] cffi default: Get rid of deprecated Python C API
 functions
Message-ID: <59c210da.d7afdf0a.8ea13.6125@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3019:0a518e0c9470
Date: 2017-09-20 08:55 +0200
http://bitbucket.org/cffi/cffi/changeset/0a518e0c9470/

Log:	Get rid of deprecated Python C API functions

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -101,7 +101,11 @@
 # define PyText_FromFormat PyUnicode_FromFormat
 # define PyText_AsUTF8 _PyUnicode_AsString   /* PyUnicode_AsUTF8 in Py3.3 */
 # define PyText_AS_UTF8 _PyUnicode_AsString
-# define PyText_GetSize PyUnicode_GetSize
+# if PY_VERSION_HEX >= 0x03030000
+#  define PyText_GetSize PyUnicode_GetLength
+# else
+#  define PyText_GetSize PyUnicode_GetSize
+# endif
 # define PyText_FromString PyUnicode_FromString
 # define PyText_FromStringAndSize PyUnicode_FromStringAndSize
 # define PyText_InternInPlace PyUnicode_InternInPlace
diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -56,14 +56,17 @@
     }
 }
 
+/* forward: from _cffi_backend.c */
+static int _fetch_as_buffer(PyObject *x, Py_buffer *view, int writable_only);
+
 static int mb_ass_slice(MiniBufferObj *self,
                         Py_ssize_t left, Py_ssize_t right, PyObject *other)
 {
-    const void *buffer;
-    Py_ssize_t buffer_len, count;
+    Py_ssize_t count;
     Py_ssize_t size = self->mb_size;
+    Py_buffer src_view;
 
-    if (PyObject_AsReadBuffer(other, &buffer, &buffer_len) < 0)
+    if (_fetch_as_buffer(other, &src_view, 0) < 0)
         return -1;
 
     if (left < 0)     left = 0;
@@ -71,12 +74,14 @@
     if (left > right) left = right;
 
     count = right - left;
-    if (count != buffer_len) {
+    if (count != src_view.len) {
+        PyBuffer_Release(&src_view);
         PyErr_SetString(PyExc_ValueError,
                         "right operand length must match slice length");
         return -1;
     }
-    memcpy(self->mb_data + left, buffer, count);
+    memcpy(self->mb_data + left, src_view.buf, count);
+    PyBuffer_Release(&src_view);
     return 0;
 }
 

From pypy.commits at gmail.com  Wed Sep 20 03:36:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 20 Sep 2017 00:36:35 -0700 (PDT)
Subject: [pypy-commit] cffi default: Only run the memory-intensive tests on
 Linux x86 or x64 machines.
Message-ID: <59c21a83.d7afdf0a.8ea13.6922@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3020:72b862bbb88e
Date: 2017-09-20 09:36 +0200
http://bitbucket.org/cffi/cffi/changeset/72b862bbb88e/

Log:	Only run the memory-intensive tests on Linux x86 or x64 machines.

diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2455,9 +2455,18 @@
     pt = lib.call2(lib.cb2)
     assert (pt.x, pt.y) == (99*500*999, -99*500*999)
 
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
 def test_ffi_gc_size_arg():
     # with PyPy's GC, these calls to ffi.gc() would rapidly consume
     # 40 GB of RAM without the third argument
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2477,6 +2486,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2294,7 +2294,16 @@
     assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
     assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
 
-def test_gc_pypy_size_arg():
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
+def test_ffi_gc_size_arg():
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2316,6 +2325,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""

From pypy.commits at gmail.com  Wed Sep 20 03:37:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 20 Sep 2017 00:37:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: import cffi/72b862bbb88e
Message-ID: <59c21ab5.196b1c0a.1fbdf.6928@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92422:8de1edd0e447
Date: 2017-09-20 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/8de1edd0e447/

Log:	import cffi/72b862bbb88e

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2456,9 +2456,18 @@
     pt = lib.call2(lib.cb2)
     assert (pt.x, pt.y) == (99*500*999, -99*500*999)
 
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
 def test_ffi_gc_size_arg():
     # with PyPy's GC, these calls to ffi.gc() would rapidly consume
     # 40 GB of RAM without the third argument
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2467,8 +2476,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
 
@@ -2478,6 +2487,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -37,7 +37,7 @@
     v = cffi.__version__.replace('+', '')
     p = os.path.join(parent, 'doc', 'source', 'installation.rst')
     content = open(p).read()
-    assert ("cffi/cffi-%s.tar.gz" % v) in content
+    assert ("/cffi-%s.tar.gz" % v) in content
 
 def test_setup_version():
     parent = os.path.dirname(os.path.dirname(cffi.__file__))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2295,7 +2295,16 @@
     assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
     assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
 
-def test_gc_pypy_size_arg():
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
+def test_ffi_gc_size_arg():
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2304,8 +2313,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
         # with PyPy's GC, the above would rapidly consume 40 GB of RAM
@@ -2317,6 +2326,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""

From pypy.commits at gmail.com  Wed Sep 20 05:42:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 20 Sep 2017 02:42:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: try harder to skip _vmprof tests on
 win32
Message-ID: <59c23805.2eb0df0a.c25d9.a464@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92423:00da64c1290f
Date: 2017-09-19 22:52 +0300
http://bitbucket.org/pypy/pypy/changeset/00da64c1290f/

Log:	try harder to skip _vmprof tests on win32

diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,5 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.rvmprof import VMProfPlatformUnsupported
+from rpython.translator.platform import CompilationError
+
 
 class Module(MixedModule):
     """
@@ -29,3 +31,9 @@
     import pypy.module._vmprof.interp_vmprof
 except VMProfPlatformUnsupported as e:
     pass
+except CompilationError as e:
+    import sys
+    if sys.platform == 'win32':
+        pass
+    else:
+        raise
diff --git a/pypy/module/_vmprof/conftest.py b/pypy/module/_vmprof/conftest.py
--- a/pypy/module/_vmprof/conftest.py
+++ b/pypy/module/_vmprof/conftest.py
@@ -1,6 +1,8 @@
-import py, platform
+import py, platform, sys
 
 def pytest_collect_directory(path, parent):
     if platform.machine() == 's390x':
-        py.test.skip("zarch tests skipped")
+        py.test.skip("_vmprof tests skipped")
+    if sys.platform == 'win32':
+        py.test.skip("_vmprof tests skipped")
 pytest_collect_file = pytest_collect_directory

From pypy.commits at gmail.com  Wed Sep 20 10:37:39 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 20 Sep 2017 07:37:39 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix test to pass on win32
Message-ID: <59c27d33.84c6df0a.8e14.c60e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92424:6a339c39b5eb
Date: 2017-09-20 17:36 +0300
http://bitbucket.org/pypy/pypy/changeset/6a339c39b5eb/

Log:	fix test to pass on win32

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -344,7 +344,7 @@
                 int recurse(void);
                 res = 0;
                 oldlimit = Py_GetRecursionLimit();
-                Py_SetRecursionLimit(200);
+                Py_SetRecursionLimit(oldlimit/100);
                 res = recurse();
                 Py_SetRecursionLimit(oldlimit);
                 if (PyErr_Occurred())

From pypy.commits at gmail.com  Wed Sep 20 13:27:07 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Wed, 20 Sep 2017 10:27:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix memory leak in _get_crl_dp
Message-ID: <59c2a4eb.a7a6df0a.6b46.13bc@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r92425:a18e6d50eca1
Date: 2017-09-20 10:26 -0700
http://bitbucket.org/pypy/pypy/changeset/a18e6d50eca1/

Log:	fix memory leak in _get_crl_dp

	see https://github.com/python/cpython/commit/b2b00e0

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -996,9 +996,6 @@
         libssl_AUTHORITY_INFO_ACCESS_free(info)
 
 def _get_crl_dp(space, certificate):
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        # Calls x509v3_cache_extensions and sets up crldp
-        libssl_X509_check_ca(certificate)
     dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
         certificate, NID_crl_distribution_points, None, None))
     if not dps:
@@ -1020,8 +1017,7 @@
                 s_uri = rffi.charpsize2str(uri.c_data, length)
                 cdp_w.append(space.newtext(s_uri))
     finally:
-        if OPENSSL_VERSION_NUMBER < 0x10001000:
-            libssl_sk_DIST_POINT_free(dps)
+        libssl_CRL_DIST_POINTS_free(dps)
     return space.newtuple(cdp_w[:])
 
 def checkwait(space, w_sock, writing):
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -473,6 +473,7 @@
 ssl_external('sk_ACCESS_DESCRIPTION_value', [AUTHORITY_INFO_ACCESS, rffi.INT], ACCESS_DESCRIPTION,
              macro=True)
 ssl_external('AUTHORITY_INFO_ACCESS_free', [AUTHORITY_INFO_ACCESS], lltype.Void)
+ssl_external('CRL_DIST_POINTS_free', [stack_st_DIST_POINT], lltype.Void)
 
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,

From pypy.commits at gmail.com  Wed Sep 20 14:33:09 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 20 Sep 2017 11:33:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Do implicit chaining in PyErr_SetObject()
Message-ID: <59c2b465.a7a6df0a.6b46.2391@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92427:eb4262e7b726
Date: 2017-09-20 19:31 +0100
http://bitbucket.org/pypy/pypy/changeset/eb4262e7b726/

Log:	Do implicit chaining in PyErr_SetObject()

diff --git a/pypy/module/cpyext/pyerrors.py b/pypy/module/cpyext/pyerrors.py
--- a/pypy/module/cpyext/pyerrors.py
+++ b/pypy/module/cpyext/pyerrors.py
@@ -50,7 +50,9 @@
     """This function is similar to PyErr_SetString() but lets you specify an
     arbitrary Python object for the "value" of the exception."""
     state = space.fromcache(State)
-    state.set_exception(OperationError(w_type, w_value))
+    operr = OperationError(w_type, w_value)
+    operr.record_context(space, space.getexecutioncontext())
+    state.set_exception(operr)
 
 @cpython_api([PyObject, CONST_STRING], lltype.Void)
 def PyErr_SetString(space, w_type, message_ptr):

From pypy.commits at gmail.com  Wed Sep 20 14:33:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 20 Sep 2017 11:33:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Save and restore sys.exc_info() in the
 test, like real code (Cython) does
Message-ID: <59c2b463.41addf0a.664df.0ada@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92426:36d12b5e3f5a
Date: 2017-09-20 19:30 +0100
http://bitbucket.org/pypy/pypy/changeset/36d12b5e3f5a/

Log:	Save and restore sys.exc_info() in the test, like real code (Cython)
	does

diff --git a/pypy/module/cpyext/test/test_exception.py b/pypy/module/cpyext/test/test_exception.py
--- a/pypy/module/cpyext/test/test_exception.py
+++ b/pypy/module/cpyext/test/test_exception.py
@@ -51,6 +51,8 @@
             ("raise_exc", "METH_NOARGS",
              """
                 PyObject *ev, *et, *tb;
+                PyObject *ev0, *et0, *tb0;
+                PyErr_GetExcInfo(&ev0, &et0, &tb0);
                 PyErr_SetString(PyExc_ValueError, "foo");
 
                 // simplified copy of __Pyx_GetException
@@ -60,6 +62,7 @@
                 PyErr_SetExcInfo(et, ev, tb);
 
                 PyErr_SetString(PyExc_TypeError, "bar");
+                PyErr_SetExcInfo(ev0, et0, tb0);
                 return NULL;
              """)])
         excinfo = raises(TypeError, module.raise_exc)

From pypy.commits at gmail.com  Wed Sep 20 23:39:37 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 20 Sep 2017 20:39:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add async slot functions
Message-ID: <59c33479.54081c0a.20522.39cf@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92428:1184b9d1641d
Date: 2017-09-21 04:39 +0100
http://bitbucket.org/pypy/pypy/changeset/1184b9d1641d/

Log:	Add async slot functions

diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -373,6 +373,7 @@
         assert 'while calling recurse' in str(excinfo.value)
 
     def test_build_class(self):
+            """
             # make sure PyObject_Call generates a proper PyTypeObject,
             # along the way verify that userslot has iter and next
             module = self.import_extension('foo', [
@@ -397,7 +398,35 @@
                         return NULL;
                     }
                     return args->ob_type->tp_iternext(args);
-                 '''),])
+                 '''),
+                ('await_', "METH_O",
+                 '''
+                    if (NULL == args->ob_type->tp_as_async->am_await)
+                    {
+                        PyErr_SetString(PyExc_TypeError, "NULL am_await");
+                        return NULL;
+                    }
+                    return args->ob_type->tp_as_async->am_await(args);
+                 '''),
+                ('aiter', "METH_O",
+                 '''
+                    if (NULL == args->ob_type->tp_as_async->am_aiter)
+                    {
+                        PyErr_SetString(PyExc_TypeError, "NULL am_aiter");
+                        return NULL;
+                    }
+                    return args->ob_type->tp_as_async->am_aiter(args);
+                 '''),
+                ('anext', "METH_O",
+                 '''
+                    if (NULL == args->ob_type->tp_as_async->am_anext)
+                    {
+                        PyErr_SetString(PyExc_TypeError, "NULL am_anext");
+                        return NULL;
+                    }
+                    return args->ob_type->tp_as_async->am_anext(args);
+                 '''),
+            ])
             def __init__(self, N):
                 self.N = N
                 self.i = 0
@@ -424,3 +453,59 @@
             except StopIteration:
                 pass
             assert out == [0, 1, 2, 3, 4]
+
+            def run_async(coro):
+                buffer = []
+                result = None
+                while True:
+                    try:
+                        buffer.append(coro.send(None))
+                    except StopIteration as ex:
+                        result = ex.value
+                        break
+                return buffer, result
+
+            def __await__(self):
+                yield 42
+                return 100
+
+            Awaitable = module.object_call((
+                'Awaitable', (object,), {'__await__': __await__}))
+
+            async def wrapper():
+                return await Awaitable()
+
+            assert run_async(module.await_(Awaitable())) == ([42], 100)
+            assert run_async(wrapper()) == ([42], 100)
+
+            def __aiter__(self):
+                return self
+
+            async def __anext__(self):
+                if self.i < self.N:
+                    res = self.i
+                    self.i += 1
+                    return res
+                raise StopAsyncIteration
+
+            AIter = module.object_call(('AIter', (object,),
+                {'__init__': __init__, '__aiter__': __aiter__,
+                 '__anext__': __anext__}))
+
+            async def list1():
+                s = []
+                async for i in AIter(3):
+                    s.append(i)
+                return s
+            async def list2():
+                s = []
+                ait = module.aiter(AIter(3))
+                try:
+                    while True:
+                        s.append(await module.anext(ait))
+                except StopAsyncIteration:
+                    return s
+
+            assert run_async(list1()) == ([], [0, 1, 2])
+            assert run_async(list2()) == ([], [0, 1, 2])
+            """
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -643,6 +643,7 @@
     pto.c_ob_pypy_link = 0
     pto.c_ob_type = metatype
     pto.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    pto.c_tp_as_async = heaptype.c_as_async
     pto.c_tp_as_number = heaptype.c_as_number
     pto.c_tp_as_sequence = heaptype.c_as_sequence
     pto.c_tp_as_mapping = heaptype.c_as_mapping
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -130,3 +130,27 @@
 @slot_function([PyObject], PyObject)
 def slot_tp_iternext(space, w_self):
     return space.next(w_self)
+
+ at slot_function([PyObject], PyObject)
+def slot_am_await(space, w_self):
+    w_await = space.lookup(w_self, "__await__")
+    if w_await is None:
+        raise oefmt(space.w_TypeError,
+            "object %T does not have __await__ method", w_self)
+    return space.get_and_call_function(w_await, w_self)
+
+ at slot_function([PyObject], PyObject)
+def slot_am_aiter(space, w_self):
+    w_aiter = space.lookup(w_self, "__aiter__")
+    if w_aiter is None:
+        raise oefmt(space.w_TypeError,
+            "object %T does not have __aiter__ method", w_self)
+    return space.get_and_call_function(w_aiter, w_self)
+
+ at slot_function([PyObject], PyObject)
+def slot_am_anext(space, w_self):
+    w_anext = space.lookup(w_self, "__anext__")
+    if w_anext is None:
+        raise oefmt(space.w_TypeError,
+            "object %T does not have __anext__ method", w_self)
+    return space.get_and_call_function(w_anext, w_self)

From pypy.commits at gmail.com  Thu Sep 21 03:01:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 00:01:09 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8: Add an elidable
 surrogate_in_utf8() function
Message-ID: <59c363b5.b5b8df0a.a478d.5299@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: unicode-utf8
Changeset: r92429:bf832917b82d
Date: 2017-09-21 08:56 +0200
http://bitbucket.org/pypy/pypy/changeset/bf832917b82d/

Log:	Add an elidable surrogate_in_utf8() function

diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -347,6 +347,16 @@
     assert pos == len(s)
     return pos - continuation_bytes
 
+ at jit.elidable
+def surrogate_in_utf8(value):
+    """Check if the UTF-8 byte string 'value' contains a surrogate.
+    The 'value' argument must be otherwise correctly formed for UTF-8.
+    """
+    for i in range(len(value) - 2):
+        if value[i] == '\xed' and value[i + 1] >= '\xa0':
+            return True
+    return False
+
 
 UTF8_INDEX_STORAGE = lltype.GcArray(lltype.Struct(
     'utf8_loc',
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -98,3 +98,10 @@
     for i in range(len(u)):
         assert (rutf8.codepoint_position_at_index(u.encode('utf8'), index, i) ==
                 len(u[:i].encode('utf8')))
+
+ at given(strategies.lists(strategies.characters()))
+def test_surrogate_in_utf8(unichars):
+    uni = u''.join(unichars).encode('utf-8')
+    result = rutf8.surrogate_in_utf8(uni)
+    expected = any(uch for uch in unichars if u'\ud800' <= uch <= u'\udfff')
+    assert result == expected

From pypy.commits at gmail.com  Thu Sep 21 03:32:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 00:32:39 -0700 (PDT)
Subject: [pypy-commit] cffi default: Prepare for the next minor release
Message-ID: <59c36b17.43001c0a.30009.54b3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3021:761502c1a738
Date: 2017-09-21 09:32 +0200
http://bitbucket.org/cffi/cffi/changeset/761502c1a738/

Log:	Prepare for the next minor release

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,12 @@
 ======================
 
 
+v1.11.1
+=======
+
+* Fix tests, remove deprecated C API usage
+
+
 v1.11
 =====
 

From pypy.commits at gmail.com  Thu Sep 21 03:40:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 00:40:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix in the PyPy-specific code: assign a
 number with ord(char), 
Message-ID: <59c36d0b.c55c1c0a.6f712.4c3f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92430:7fe0041fccaa
Date: 2017-09-21 09:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7fe0041fccaa/

Log:	Fix in the PyPy-specific code: assign a number with ord(char), to
	allow 'tostr' to be a unicode string too.

diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 

From pypy.commits at gmail.com  Thu Sep 21 03:58:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 00:58:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: Improve the error message for
 ``bytearray[index] = unknown_type''
Message-ID: <59c3713a.02c51c0a.cbad3.6122@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92431:4a1516a142a3
Date: 2017-09-21 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/4a1516a142a3/

Log:	Improve the error message for ``bytearray[index] = unknown_type''

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1442,7 +1442,7 @@
             length = 1
         return start, stop, step, length
 
-    def getindex_w(self, w_obj, w_exception, objdescr=None):
+    def getindex_w(self, w_obj, w_exception, objdescr=None, errmsg=None):
         """Return w_obj.__index__() as an RPython int.
         If w_exception is None, silently clamp in case of overflow;
         else raise w_exception.
@@ -1452,8 +1452,10 @@
         except OperationError as err:
             if objdescr is None or not err.match(self, self.w_TypeError):
                 raise
-            raise oefmt(self.w_TypeError, "%s must be an integer, not %T",
-                        objdescr, w_obj)
+            if errmsg is None:
+                errmsg = " must be an integer"
+            raise oefmt(self.w_TypeError, "%s%s, not %T",
+                        objdescr, errmsg, w_obj)
         try:
             # allow_conversion=False it's not really necessary because the
             # return type of __index__ is already checked by space.index(),
@@ -1690,7 +1692,8 @@
             if len(string) != 1:
                 raise oefmt(self.w_ValueError, "string must be of size 1")
             return string[0]
-        value = self.getindex_w(w_obj, None)
+        value = self.getindex_w(w_obj, None, "",
+                                "an integer or string of size 1 is required")
         if not 0 <= value < 256:
             # this includes the OverflowError in case the long is too large
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -658,10 +658,10 @@
         else:
             self.setitem(w_obj, self.newtext(key), w_value)
 
-    def getindex_w(self, w_obj, w_exception, objdescr=None):
+    def getindex_w(self, w_obj, w_exception, objdescr=None, errmsg=None):
         if type(w_obj) is W_IntObject:
             return w_obj.intval
-        return ObjSpace.getindex_w(self, w_obj, w_exception, objdescr)
+        return ObjSpace.getindex_w(self, w_obj, w_exception, objdescr, errmsg)
 
     def unicode_from_object(self, w_obj):
         from pypy.objspace.std.unicodeobject import unicode_from_object
diff --git a/pypy/objspace/std/test/test_bytearrayobject.py b/pypy/objspace/std/test/test_bytearrayobject.py
--- a/pypy/objspace/std/test/test_bytearrayobject.py
+++ b/pypy/objspace/std/test/test_bytearrayobject.py
@@ -496,6 +496,15 @@
         b[1] = 'B'
         assert b == 'aBcdefghi'
 
+    def test_setitem_errmsg(self):
+        b = bytearray('abcdefghi')
+        e = raises(TypeError, "b[1] = u'B'")
+        assert str(e.value).startswith(
+            "an integer or string of size 1 is required")
+        e = raises(TypeError, "b[1] = None")
+        assert str(e.value).startswith(
+            "an integer or string of size 1 is required")
+
     def test_setitem_slice(self):
         b = bytearray('abcdefghi')
         b[0:3] = 'ABC'

From pypy.commits at gmail.com  Thu Sep 21 11:20:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 08:20:12 -0700 (PDT)
Subject: [pypy-commit] cffi default: Fix the remaining places for PyPy3.
 (Originally in Pull Request #82)
Message-ID: <59c3d8ac.93aa1c0a.36e2d.ea59@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3022:ee67e4bc2501
Date: 2017-09-21 17:20 +0200
http://bitbucket.org/cffi/cffi/changeset/ee67e4bc2501/

Log:	Fix the remaining places for PyPy3. (Originally in Pull Request #82)

diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2497,8 +2497,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2516,8 +2516,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2336,8 +2336,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2355,8 +2355,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x

From pypy.commits at gmail.com  Thu Sep 21 11:29:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 08:29:58 -0700 (PDT)
Subject: [pypy-commit] cffi default: Some more module-like behavior hacks
Message-ID: <59c3daf6.8298df0a.96457.db1d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3023:19ab3ca30f71
Date: 2017-09-21 17:29 +0200
http://bitbucket.org/cffi/cffi/changeset/19ab3ca30f71/

Log:	Some more module-like behavior hacks

diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -530,6 +530,13 @@
         PyErr_Clear();
         return PyText_FromFormat("%s.lib", PyText_AS_UTF8(lib->l_libname));
     }
+#if PY_MAJOR_VERSION >= 3
+    if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) {
+        /* some more module-like behavior hacks */
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+#endif
     return NULL;
 }
 

From pypy.commits at gmail.com  Thu Sep 21 11:32:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 21 Sep 2017 08:32:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2652
Message-ID: <59c3db7e.44231c0a.14fbf.dcba@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92432:6f2a78fc53a2
Date: 2017-09-21 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/6f2a78fc53a2/

Log:	Issue #2652

	Trying to give the cffi lib modules some more module-like attributes

diff --git a/pypy/module/_cffi_backend/lib_obj.py b/pypy/module/_cffi_backend/lib_obj.py
--- a/pypy/module/_cffi_backend/lib_obj.py
+++ b/pypy/module/_cffi_backend/lib_obj.py
@@ -203,6 +203,10 @@
                     return self.space.gettypeobject(Module.typedef)
                 if is_getattr and attr == '__name__':
                     return self.space.newtext("%s.lib" % self.libname)
+                if is_getattr and attr == '__loader__':
+                    return self.space.w_None    # PyPy/CPython 3.x only
+                if is_getattr and attr == '__spec__':
+                    return self.space.w_None    # PyPy/CPython 3.x only
                 raise oefmt(self.space.w_AttributeError,
                             "cffi library '%s' has no function, constant "
                             "or global variable named '%s'",

From pypy.commits at gmail.com  Thu Sep 21 14:32:14 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 21 Sep 2017 11:32:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59c405ae.318fdf0a.5fa2b.4587@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92433:a42b82e1eb73
Date: 2017-09-21 19:32 +0100
http://bitbucket.org/pypy/pypy/changeset/a42b82e1eb73/

Log:	hg merge default

diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -120,7 +120,7 @@
 Download the versions of all the external packages from
 https://bitbucket.org/pypy/pypy/downloads/local_59.zip
 (for post-5.8 builds) with sha256 checksum
-``0f96c045db1f5f73ad0fae7857caa69c261324bd8e51f6d2ad1fa842c4a5f26f``
+``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
 (to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -419,3 +419,7 @@
         def f():
             return x
         assert f.__closure__[0].cell_contents is x
+
+    def test_get_with_none_arg(self):
+        raises(TypeError, type.__dict__['__mro__'].__get__, None)
+        raises(TypeError, type.__dict__['__mro__'].__get__, None, None)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -300,6 +300,8 @@
         if (space.is_w(w_obj, space.w_None)
             and not space.is_w(w_cls, space.type(space.w_None))):
             #print self, w_obj, w_cls
+            if space.is_w(w_cls, space.w_None):
+                raise oefmt(space.w_TypeError, "__get__(None, None) is invalid")
             return self
         else:
             try:
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -111,6 +111,7 @@
         raise wrap_oserror(space, rwin32.lastSavedWindowsError())
 
 def GetLastError(space):
+    """NOTE: don't use this.  See issue #2658"""
     return space.newint(rwin32.GetLastError_saved())
 
 # __________________________________________________________
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,5 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.rvmprof import VMProfPlatformUnsupported
+from rpython.translator.platform import CompilationError
+
 
 class Module(MixedModule):
     """
@@ -29,3 +31,9 @@
     import pypy.module._vmprof.interp_vmprof
 except VMProfPlatformUnsupported as e:
     pass
+except CompilationError as e:
+    import sys
+    if sys.platform == 'win32':
+        pass
+    else:
+        raise
diff --git a/pypy/module/_vmprof/conftest.py b/pypy/module/_vmprof/conftest.py
--- a/pypy/module/_vmprof/conftest.py
+++ b/pypy/module/_vmprof/conftest.py
@@ -1,6 +1,8 @@
-import py, platform
+import py, platform, sys
 
 def pytest_collect_directory(path, parent):
     if platform.machine() == 's390x':
-        py.test.skip("zarch tests skipped")
+        py.test.skip("_vmprof tests skipped")
+    if sys.platform == 'win32':
+        py.test.skip("_vmprof tests skipped")
 pytest_collect_file = pytest_collect_directory
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -102,21 +102,27 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
+        w_objclass = self.w_objclass 
+        assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
-            self.name, self.w_objclass.getname(self.space).encode('utf-8')))
+            self.name, w_objclass.name))
 
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
-                "descriptor '%8' of '%N' object needs an argument",
-                self.name, self.w_objclass)
+                "descriptor '%8' of '%s' object needs an argument",
+                self.name, w_objclass.name)
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
+            w_objclass = self.w_objclass 
+            assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
-                "descriptor '%8' requires a '%N' object but received a '%T'",
-                self.name, self.w_objclass, w_instance)
+                "descriptor '%8' requires a '%s' object but received a '%T'",
+                self.name, w_objclass.name, w_instance)
         w_args = space.newtuple(args_w[1:])
         w_kw = space.newdict()
         for key, w_obj in kw_w.items():
diff --git a/pypy/module/cpyext/test/test_eval.py b/pypy/module/cpyext/test/test_eval.py
--- a/pypy/module/cpyext/test/test_eval.py
+++ b/pypy/module/cpyext/test/test_eval.py
@@ -352,7 +352,7 @@
                 int recurse(void);
                 res = 0;
                 oldlimit = Py_GetRecursionLimit();
-                Py_SetRecursionLimit(200);
+                Py_SetRecursionLimit(oldlimit/100);
                 res = recurse();
                 Py_SetRecursionLimit(oldlimit);
                 if (PyErr_Occurred())
diff --git a/pypy/module/sys/__init__.py b/pypy/module/sys/__init__.py
--- a/pypy/module/sys/__init__.py
+++ b/pypy/module/sys/__init__.py
@@ -92,8 +92,6 @@
         'int_info'              : 'system.get_int_info(space)',
         'hash_info'             : 'system.get_hash_info(space)',
         'float_repr_style'      : 'system.get_float_repr_style(space)',
-        'getdlopenflags'        : 'system.getdlopenflags',
-        'setdlopenflags'        : 'system.setdlopenflags',
 
         'get_coroutine_wrapper' : 'vm.get_coroutine_wrapper',
         'set_coroutine_wrapper' : 'vm.set_coroutine_wrapper',
@@ -104,6 +102,9 @@
     if sys.platform == 'win32':
         interpleveldefs['winver'] = 'version.get_winver(space)'
         interpleveldefs['getwindowsversion'] = 'vm.getwindowsversion'
+    else:
+        interpleveldefs['getdlopenflags'] = 'system.getdlopenflags'
+        interpleveldefs['setdlopenflags'] = 'system.setdlopenflags'
 
     appleveldefs = {
         'excepthook'            : 'app.excepthook',
diff --git a/pypy/module/sys/test/test_sysmodule.py b/pypy/module/sys/test/test_sysmodule.py
--- a/pypy/module/sys/test/test_sysmodule.py
+++ b/pypy/module/sys/test/test_sysmodule.py
@@ -492,6 +492,8 @@
 
     def test_dlopenflags(self):
         import sys
+        if not hasattr(sys, "getdlopenflags"):
+            skip('{gs}etdlopenflags is not implemented on this platform')
         raises(TypeError, sys.getdlopenflags, 42)
         oldflags = sys.getdlopenflags()
         raises(TypeError, sys.setdlopenflags)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2456,9 +2456,18 @@
     pt = lib.call2(lib.cb2)
     assert (pt.x, pt.y) == (99*500*999, -99*500*999)
 
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
 def test_ffi_gc_size_arg():
     # with PyPy's GC, these calls to ffi.gc() would rapidly consume
     # 40 GB of RAM without the third argument
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2467,8 +2476,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
 
@@ -2478,6 +2487,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_version.py
@@ -37,7 +37,7 @@
     v = cffi.__version__.replace('+', '')
     p = os.path.join(parent, 'doc', 'source', 'installation.rst')
     content = open(p).read()
-    assert ("cffi/cffi-%s.tar.gz" % v) in content
+    assert ("/cffi-%s.tar.gz" % v) in content
 
 def test_setup_version():
     parent = os.path.dirname(os.path.dirname(cffi.__file__))
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1,7 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 
 import sys, os, py
-from cffi import FFI, VerificationError, FFIError
+from cffi import FFI, VerificationError, FFIError, CDefError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import u, long
@@ -1127,7 +1127,9 @@
 
 def test_some_float_invalid_1():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, "typedef long double... foo_t;")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, "typedef long double... foo_t;")
 
 def test_some_float_invalid_2():
     ffi = FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -1,6 +1,7 @@
 # Generated by pypy/tool/import_cffi.py
 import os, sys, math, py
 from cffi import FFI, FFIError, VerificationError, VerificationMissing, model
+from cffi import CDefError
 from cffi import recompiler
 from pypy.module.test_lib_pypy.cffi_tests.support import *
 import _cffi_backend
@@ -2223,7 +2224,9 @@
 
 def test_unsupported_some_primitive_types():
     ffi = FFI()
-    py.test.raises(FFIError, ffi.cdef, """typedef void... foo_t;""")
+    py.test.raises((FFIError,      # with pycparser <= 2.17
+                    CDefError),    # with pycparser >= 2.18
+                   ffi.cdef, """typedef void... foo_t;""")
     #
     ffi.cdef("typedef int... foo_t;")
     py.test.raises(VerificationError, ffi.verify, "typedef float foo_t;")
@@ -2292,7 +2295,16 @@
     assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
     assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
 
-def test_gc_pypy_size_arg():
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
+def test_ffi_gc_size_arg():
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2301,8 +2313,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
         # with PyPy's GC, the above would rapidly consume 40 GB of RAM
@@ -2314,6 +2326,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
diff --git a/rpython/rlib/ropenssl.py b/rpython/rlib/ropenssl.py
--- a/rpython/rlib/ropenssl.py
+++ b/rpython/rlib/ropenssl.py
@@ -473,6 +473,7 @@
 ssl_external('sk_ACCESS_DESCRIPTION_value', [AUTHORITY_INFO_ACCESS, rffi.INT], ACCESS_DESCRIPTION,
              macro=True)
 ssl_external('AUTHORITY_INFO_ACCESS_free', [AUTHORITY_INFO_ACCESS], lltype.Void)
+ssl_external('CRL_DIST_POINTS_free', [stack_st_DIST_POINT], lltype.Void)
 
 ssl_external('GENERAL_NAME_print', [BIO, GENERAL_NAME], rffi.INT)
 ssl_external('pypy_GENERAL_NAME_dirn', [GENERAL_NAME], X509_NAME,

From pypy.commits at gmail.com  Thu Sep 21 21:35:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 21 Sep 2017 18:35:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add async slots to built-in types
Message-ID: <59c468f7.75afdf0a.17675.8698@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92434:5dd50480d9b7
Date: 2017-09-22 02:35 +0100
http://bitbucket.org/pypy/pypy/changeset/5dd50480d9b7/

Log:	Add async slots to built-in types

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -442,7 +442,10 @@
 
     handled = False
     # unary functions
-    for tp_name, attr in [('tp_as_number.c_nb_int', '__int__'),
+    for tp_name, attr in [('tp_as_async.c_am_await', '__await__'),
+                          ('tp_as_async.c_am_anext', '__anext__'),
+                          ('tp_as_async.c_am_aiter', '__aiter__'),
+                          ('tp_as_number.c_nb_int', '__int__'),
                           ('tp_as_number.c_nb_long', '__long__'),
                           ('tp_as_number.c_nb_float', '__float__'),
                           ('tp_as_number.c_nb_negative', '__neg__'),
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
--- a/pypy/module/cpyext/test/test_genobject.py
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -1,3 +1,4 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.genobject import PyGen_Check, PyGen_CheckExact
 from pypy.module.cpyext.genobject import PyCoro_CheckExact
@@ -25,3 +26,21 @@
         assert not PyGen_Check(space, w_coroutine)
         assert not PyGen_CheckExact(space, w_coroutine)
         assert PyCoro_CheckExact(space, w_coroutine)
+
+class AppTestCoroutine(AppTestCpythonExtensionBase):
+    def test_simple(self):
+        """
+        module = self.import_extension('test_coroutine', [
+            ('await_', 'METH_O',
+             '''
+             PyAsyncMethods* am = args->ob_type->tp_as_async;
+             if (am && am->am_await) {
+                return am->am_await(args);
+             }
+             PyErr_SetString(PyExc_TypeError, "Not an awaitable");
+             return NULL;
+             '''),])
+        async def f():
+            pass
+        raises(StopIteration, next, module.await_(f()))
+        """

From pypy.commits at gmail.com  Fri Sep 22 08:39:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 05:39:03 -0700 (PDT)
Subject: [pypy-commit] pypy unicode-utf8:
 rutf8.codepoint_position_at_index() should also work for index == len(u)
Message-ID: <59c50467.cad81c0a.38caf.e603@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: unicode-utf8
Changeset: r92435:0f1073a0843b
Date: 2017-09-22 14:37 +0200
http://bitbucket.org/pypy/pypy/changeset/0f1073a0843b/

Log:	rutf8.codepoint_position_at_index() should also work for index ==
	len(u)

diff --git a/rpython/rlib/rutf8.py b/rpython/rlib/rutf8.py
--- a/rpython/rlib/rutf8.py
+++ b/rpython/rlib/rutf8.py
@@ -95,6 +95,8 @@
     """
     pos = r_uint(pos)
     pos -= 1
+    if pos >= len(code):     # for the case where pos - 1 == len(code):
+        return pos           # assume there is an extra '\x00' character
     chr1 = ord(code[pos])
     if chr1 <= 0x7F:
         return pos
@@ -377,9 +379,9 @@
     """ Create an index storage which stores index of each 4th character
     in utf8 encoded unicode string.
     """
-    if len(utf8) == utf8len <= ASCII_INDEX_STORAGE_BLOCKS * 64:
+    if len(utf8) == utf8len < ASCII_INDEX_STORAGE_BLOCKS * 64:
         return ASCII_INDEX_STORAGE
-    arraysize = (utf8len + 63) // 64
+    arraysize = utf8len // 64 + 1
     storage = lltype.malloc(UTF8_INDEX_STORAGE, arraysize)
     baseindex = 0
     current = 0
@@ -387,10 +389,14 @@
         storage[current].baseindex = baseindex
         next = baseindex
         for i in range(16):
-            next = next_codepoint_pos(utf8, next)
+            if utf8len == 0:
+                next += 1      # assume there is an extra '\x00' character
+            else:
+                next = next_codepoint_pos(utf8, next)
             storage[current].ofs[i] = chr(next - baseindex)
             utf8len -= 4
-            if utf8len <= 0:
+            if utf8len < 0:
+                assert current + 1 == len(storage)
                 break
             next = next_codepoint_pos(utf8, next)
             next = next_codepoint_pos(utf8, next)
diff --git a/rpython/rlib/test/test_rutf8.py b/rpython/rlib/test/test_rutf8.py
--- a/rpython/rlib/test/test_rutf8.py
+++ b/rpython/rlib/test/test_rutf8.py
@@ -93,9 +93,11 @@
                 ord(item))
 
 @given(strategies.text())
+ at example(u'x' * 64 * 5)
+ at example(u'x' * (64 * 5 - 1))
 def test_codepoint_position_at_index(u):
     index = rutf8.create_utf8_index_storage(u.encode('utf8'), len(u))
-    for i in range(len(u)):
+    for i in range(len(u) + 1):
         assert (rutf8.codepoint_position_at_index(u.encode('utf8'), index, i) ==
                 len(u[:i].encode('utf8')))
 

From pypy.commits at gmail.com  Fri Sep 22 08:57:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 05:57:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Test and fix for CPython's test_itertools
 (broken by accident in 17c8c1d27c41)
Message-ID: <59c508d2.12a4df0a.0e1f.d31d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92436:db294f23903a
Date: 2017-09-22 14:56 +0200
http://bitbucket.org/pypy/pypy/changeset/db294f23903a/

Log:	Test and fix for CPython's test_itertools (broken by accident in
	17c8c1d27c41)

diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py
--- a/pypy/module/itertools/interp_itertools.py
+++ b/pypy/module/itertools/interp_itertools.py
@@ -445,6 +445,7 @@
         __iter__ = interp2app(W_ISlice.iter_w),
         __next__ = interp2app(W_ISlice.next_w),
         __reduce__ = interp2app(W_ISlice.descr_reduce),
+        __setstate__ = interp2app(W_ISlice.descr_setstate),
         __doc__  = """Make an iterator that returns selected elements from the
     iterable.  If start is non-zero, then elements from the iterable
     are skipped until start is reached. Afterward, elements are
diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py
--- a/pypy/module/itertools/test/test_itertools.py
+++ b/pypy/module/itertools/test/test_itertools.py
@@ -1028,6 +1028,8 @@
                 assert list(itertools.islice(c2, 3)) == expected
                 c3 = pickle.loads(pickle.dumps(c))
                 assert list(itertools.islice(c3, 3)) == expected
+        c4 = copy.copy(itertools.islice([1, 2, 3], 1, 5))
+        assert list(c4) == [2, 3]
 
     def test_islice_attack(self):
         import itertools

From pypy.commits at gmail.com  Fri Sep 22 09:14:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:14:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip one check on pypy
Message-ID: <59c50cc2.338edf0a.fb511.e891@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92437:115c47f3c022
Date: 2017-09-22 15:14 +0200
http://bitbucket.org/pypy/pypy/changeset/115c47f3c022/

Log:	Skip one check on pypy

diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -348,7 +348,8 @@
     strobj = "abcde"*3
     dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
 
-    def helper3(self, rsample, recursive=False, simple=False):
+    def helper3(self, rsample, recursive=False, simple=False,
+                check_sharing=True):
         #we have two instances
         sample = (rsample, rsample)
 
@@ -358,7 +359,9 @@
         n3 = CollectObjectIDs(set(), marshal.loads(s3))
 
         #same number of instances generated
-        self.assertEqual(n3, n0)
+        # except in one corner case on top of pypy, for code objects
+        if check_sharing:
+            self.assertEqual(n3, n0)
 
         if not recursive:
             #can compare with version 2
@@ -395,7 +398,7 @@
         if __file__.endswith(".py"):
             code = compile(code, __file__, "exec")
         self.helper(code)
-        self.helper3(code)
+        self.helper3(code, check_sharing=support.check_impl_detail())
 
     def testRecursion(self):
         d = dict(self.dictobj)

From pypy.commits at gmail.com  Fri Sep 22 09:18:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:18:55 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Skip more tests on pypy
Message-ID: <59c50dbf.41addf0a.19b61.c70a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92438:8246644e701d
Date: 2017-09-22 15:17 +0200
http://bitbucket.org/pypy/pypy/changeset/8246644e701d/

Log:	Skip more tests on pypy

diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -349,7 +349,7 @@
     dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
 
     def helper3(self, rsample, recursive=False, simple=False,
-                check_sharing=True):
+                check_sharing=True, check_non_sharing=True):
         #we have two instances
         sample = (rsample, rsample)
 
@@ -368,21 +368,26 @@
             s2 = marshal.dumps(sample, 2)
             n2 = CollectObjectIDs(set(), marshal.loads(s2))
             #old format generated more instances
-            self.assertGreater(n2, n0)
+            # except on pypy where equal ints or floats always have
+            # the same id anyway
+            if check_non_sharing:
+                self.assertGreater(n2, n0)
 
             #if complex objects are in there, old format is larger
-            if not simple:
+            if check_non_sharing and not simple:
                 self.assertGreater(len(s2), len(s3))
             else:
                 self.assertGreaterEqual(len(s2), len(s3))
 
     def testInt(self):
         self.helper(self.intobj)
-        self.helper3(self.intobj, simple=True)
+        self.helper3(self.intobj, simple=True,
+                     check_non_sharing=support.check_impl_detail())
 
     def testFloat(self):
         self.helper(self.floatobj)
-        self.helper3(self.floatobj)
+        self.helper3(self.floatobj,
+                     check_non_sharing=support.check_impl_detail())
 
     def testStr(self):
         self.helper(self.strobj)

From pypy.commits at gmail.com  Fri Sep 22 09:20:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:20:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix the last test in test_marshal
Message-ID: <59c50e18.499edf0a.590ef.fe6f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92439:ea46b2e0a190
Date: 2017-09-22 15:20 +0200
http://bitbucket.org/pypy/pypy/changeset/ea46b2e0a190/

Log:	Fix the last test in test_marshal

diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -271,6 +271,11 @@
                 if n is not None and n > 4:
                     n += 10**6
                 return n
+            def read(self, n):   # PyPy calls read(), not readinto()
+                result = super().read(n)
+                if len(result) > 4:
+                    result += b'\x00' * (10**6)
+                return result
         for value in (1.0, 1j, b'0123456789', '0123456789'):
             self.assertRaises(ValueError, marshal.load,
                               BadReader(marshal.dumps(value)))

From pypy.commits at gmail.com  Fri Sep 22 09:35:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:35:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix this test,
 which might be confused by partially-initialized codecs
Message-ID: <59c511a9.e186df0a.1fec0.a28d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92440:1dbe3599b597
Date: 2017-09-22 15:35 +0200
http://bitbucket.org/pypy/pypy/changeset/1dbe3599b597/

Log:	Fix this test, which might be confused by partially-initialized
	codecs

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -928,8 +928,8 @@
     def test_constant_of_value_unknown_to_the_compiler(self):
         extra_c_source = self.udir + self.os_sep + (
             'extra_test_constant_of_value_unknown_to_the_compiler.c')
-        with open(extra_c_source, 'w') as f:
-            f.write('const int external_foo = 42;\n')
+        with open(extra_c_source, 'wb') as f:
+            f.write(b'const int external_foo = 42;\n')
         ffi, lib = self.prepare(
             "const int external_foo;",
             'test_constant_of_value_unknown_to_the_compiler',

From pypy.commits at gmail.com  Fri Sep 22 09:38:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:38:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: import cffi/19ab3ca30f71
Message-ID: <59c5125b.43001c0a.e17de.0117@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92441:5d1a2179a4b2
Date: 2017-09-22 15:37 +0200
http://bitbucket.org/pypy/pypy/changeset/5d1a2179a4b2/

Log:	import cffi/19ab3ca30f71

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2498,8 +2498,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2517,8 +2517,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2337,8 +2337,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2356,8 +2356,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x

From pypy.commits at gmail.com  Fri Sep 22 09:38:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:38:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Point to explanation for the missing
 '__builtins__' support in PyPy
Message-ID: <59c5125c.8298df0a.96457.18d1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92442:1e266d4f0472
Date: 2017-09-22 15:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1e266d4f0472/

Log:	Point to explanation for the missing '__builtins__' support in PyPy

diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -429,7 +429,8 @@
 
 * the ``__builtins__`` name is always referencing the ``__builtin__`` module,
   never a dictionary as it sometimes is in CPython. Assigning to
-  ``__builtins__`` has no effect.
+  ``__builtins__`` has no effect.  (For usages of tools like
+  RestrictedPython, see `issue #2653`_.)
 
 * directly calling the internal magic methods of a few built-in types
   with invalid arguments may have a slightly different result.  For
@@ -543,4 +544,4 @@
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
-
+.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/

From pypy.commits at gmail.com  Fri Sep 22 09:43:13 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 22 Sep 2017 06:43:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix the leakfinder issues in
 test_recompiler
Message-ID: <59c51371.908a1c0a.da65c.021b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92443:f368583e1d76
Date: 2017-09-22 15:42 +0200
http://bitbucket.org/pypy/pypy/changeset/f368583e1d76/

Log:	Fix the leakfinder issues in test_recompiler

diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -95,6 +95,7 @@
 
     def setup_method(self, meth):
         self._w_modules = self.space.appexec([], """():
+            import cpyext      # ignore stuff there in the leakfinder
             import sys
             return set(sys.modules)
         """)

From pypy.commits at gmail.com  Fri Sep 22 15:20:23 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 22 Sep 2017 12:20:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: cache the string keys that occur in the
 json dicts, as they are likely to repeat
Message-ID: <59c56277.919bdf0a.63f17.0d35@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92444:8aeaf30c80e8
Date: 2017-09-22 21:18 +0200
http://bitbucket.org/pypy/pypy/changeset/8aeaf30c80e8/

Log:	cache the string keys that occur in the json dicts, as they are
	likely to repeat

	this reduces both parsing time and memory usage

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize, always_inline
+from rpython.rlib.objectmodel import specialize, always_inline, r_dict
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -42,6 +42,22 @@
         ll_res.chars[i] = cast_primitive(UniChar, ch)
     return hlunicode(ll_res)
 
+def slice_eq(a, b):
+    (ll_chars1, start1, length1, _) = a
+    (ll_chars2, start2, length2, _) = b
+    if length1 != length2:
+        return False
+    j = start2
+    for i in range(start1, start1 + length1):
+        if ll_chars1[i] != ll_chars2[j]:
+            return False
+        j += 1
+    return True
+
+def slice_hash(a):
+    (ll_chars, start, length, h) = a
+    return h
+
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 class JSONDecoder(object):
@@ -55,7 +71,7 @@
         self.ll_chars = rffi.str2charp(s)
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
-        self.last_type = TYPE_UNKNOWN
+        self.cache = r_dict(slice_eq, slice_hash)
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -295,22 +311,23 @@
             i += 1
             bits |= ord(ch)
             if ch == '"':
-                if bits & 0x80:
-                    # the 8th bit is set, it's an utf8 strnig
-                    content_utf8 = self.getslice(start, i-1)
-                    content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
-                else:
-                    # ascii only, fast path (ascii is a strict subset of
-                    # latin1, and we already checked that all the chars are <
-                    # 128)
-                    content_unicode = strslice2unicode_latin1(self.s, start, i-1)
-                self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.newunicode(content_unicode)
+                return self.space.newunicode(
+                        self._create_string(start, i - 1, bits))
             elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
                 return self.decode_string_escaped(start)
 
+    def _create_string(self, start, end, bits):
+        if bits & 0x80:
+            # the 8th bit is set, it's an utf8 string
+            content_utf8 = self.getslice(start, end)
+            return unicodehelper.decode_utf8(self.space, content_utf8)
+        else:
+            # ascii only, fast path (ascii is a strict subset of
+            # latin1, and we already checked that all the chars are <
+            # 128)
+            return strslice2unicode_latin1(self.s, start, end)
 
     def decode_string_escaped(self, start):
         i = self.pos
@@ -324,7 +341,6 @@
             if ch == '"':
                 content_utf8 = builder.build()
                 content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
-                self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.newunicode(content_unicode)
             elif ch == '\\':
@@ -387,6 +403,48 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
+    def decode_key(self, i):
+        """ returns an unwrapped unicode """
+        from rpython.rlib.rarithmetic import intmask
+
+        i = self.skip_whitespace(i)
+        ll_chars = self.ll_chars
+        ch = ll_chars[i]
+        if ch != '"':
+            self._raise("Key name must be string at char %d", i)
+        i += 1
+
+        start = i
+        bits = 0
+        strhash = ord(ll_chars[i]) << 7
+        while True:
+            ch = ll_chars[i]
+            i += 1
+            if ch == '"':
+                break
+            elif ch == '\\' or ch < '\x20':
+                self.pos = i-1
+                return self.space.unicode_w(self.decode_string_escaped(start))
+            strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i]))
+            bits |= ord(ch)
+        length = i - start - 1
+        if length == 0:
+            strhash = -1
+        else:
+            strhash ^= length
+            strhash = intmask(strhash)
+        self.pos = i
+        # check cache first:
+        key = (ll_chars, start, length, strhash)
+        try:
+            return self.cache[key]
+        except KeyError:
+            pass
+        res = self._create_string(start, i - 1, bits)
+        self.cache[key] = res
+        return res
+
+
 def loads(space, w_s):
     if space.isinstance_w(w_s, space.w_unicode):
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,7 +10,18 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+def test_decode_key():
+    s1 = "123" * 100
+    s = ' "%s"   "%s" ' % (s1, s1)
+    dec = JSONDecoder('fake space', s)
+    assert dec.pos == 0
+    x = dec.decode_key(0)
+    assert x == s1
+    # check caching
+    y = dec.decode_key(dec.pos)
+    assert y == s1
+    assert y is x
+    dec.close()
 
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
@@ -190,6 +201,12 @@
         res = _pypyjson.loads(json)
         assert res == {u'a': u'\ud83d'}
 
+    def test_cache_keys(self):
+        import _pypyjson
+        json = '[{"a": 1}, {"a": 2}]'
+        res = _pypyjson.loads(json)
+        assert res == [{u'a': 1}, {u'a': 2}]
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
@@ -226,7 +243,7 @@
             ('{"spam":[42}', "Unexpected '}' when decoding array (char 11)"),
             ('["]', 'Unterminated string starting at char 1'),
             ('["spam":', "Unexpected ':' when decoding array (char 7)"),
-            ('[{]', "No JSON object could be decoded: unexpected ']' at char 2"),
+            ('[{]', "Key name must be string at char 2"),
         ]
         for inputtext, errmsg in test_cases:
             exc = raises(ValueError, _pypyjson.loads, inputtext)

From pypy.commits at gmail.com  Fri Sep 22 15:20:24 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Fri, 22 Sep 2017 12:20:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: create a dict with the unicode strategy
 directly
Message-ID: <59c56278.64a8df0a.6fb14.610c@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92445:64e7df28f623
Date: 2017-09-22 21:19 +0200
http://bitbucket.org/pypy/pypy/changeset/64e7df28f623/

Log:	create a dict with the unicode strategy directly

	(also fix targetjson)

diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -264,19 +264,16 @@
 
     def decode_object(self, i):
         start = i
-        w_dict = self.space.newdict()
-        #
+
         i = self.skip_whitespace(i)
         if self.ll_chars[i] == '}':
             self.pos = i+1
-            return w_dict
-        #
+            return self.space.newdict()
+
+        d = {}
         while True:
             # parse a key: value
-            self.last_type = TYPE_UNKNOWN
-            w_name = self.decode_any(i)
-            if self.last_type != TYPE_STRING:
-                self._raise("Key name must be string for object starting at char %d", start)
+            name = self.decode_key(i)
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
@@ -285,13 +282,13 @@
             i = self.skip_whitespace(i)
             #
             w_value = self.decode_any(i)
-            self.space.setitem(w_dict, w_name, w_value)
+            d[name] = w_value
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             i += 1
             if ch == '}':
                 self.pos = i
-                return w_dict
+                return self._create_dict(d)
             elif ch == ',':
                 pass
             elif ch == '\0':
@@ -300,6 +297,9 @@
                 self._raise("Unexpected '%s' when decoding object (char %d)",
                             ch, i-1)
 
+    def _create_dict(self, d):
+        from pypy.objspace.std.dictmultiobject import from_unicode_key_dict
+        return from_unicode_key_dict(self.space, d)
 
     def decode_string(self, i):
         start = i
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -5,9 +5,15 @@
 
 import time
 from pypy.interpreter.error import OperationError
-from pypy.module._pypyjson.interp_decoder import loads
+from pypy.module._pypyjson.interp_decoder import loads, JSONDecoder
 from rpython.rlib.objectmodel import specialize, dont_inline
 
+def _create_dict(self, d):
+    w_res = W_Dict()
+    w_res.dictval = d
+    return w_res
+
+JSONDecoder._create_dict = _create_dict
 
 ## MSG = open('msg.json').read()
 
@@ -65,10 +71,14 @@
     def isinstance_w(self, w_x, w_type):
         return isinstance(w_x, w_type)
 
-    def str_w(self, w_x):
+    def bytes_w(self, w_x):
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    def unicode_w(self, w_x):
+        assert isinstance(w_x, W_Unicode)
+        return w_x.unival
+
     @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
@@ -83,13 +93,17 @@
         assert isinstance(key, W_Unicode)
         d.dictval[key.unival] = value
 
-    def wrapunicode(self, x):
+    def newunicode(self, x):
         return W_Unicode(x)
 
-    def wrapint(self, x):
+    def newtext(self, x):
+        return W_String(x)
+    newbytes = newtext
+
+    def newint(self, x):
         return W_Int(x)
 
-    def wrapfloat(self, x):
+    def newfloat(self, x):
         return W_Float(x)
 
     @specialize.argtype(1)
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1257,6 +1257,12 @@
 create_iterator_classes(UnicodeDictStrategy)
 
 
+def from_unicode_key_dict(space, d):
+    strategy = space.fromcache(UnicodeDictStrategy)
+    storage = strategy.erase(d)
+    return W_DictObject(space, strategy, storage)
+
+
 class IntDictStrategy(AbstractTypedStrategy, DictStrategy):
     erase, unerase = rerased.new_erasing_pair("int")
     erase = staticmethod(erase)

From pypy.commits at gmail.com  Sat Sep 23 07:41:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 23 Sep 2017 04:41:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix for ctypes.cast(bytes)
Message-ID: <59c6484d.75afdf0a.ea78.3326@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92446:a600b93a7578
Date: 2017-09-23 13:40 +0200
http://bitbucket.org/pypy/pypy/changeset/a600b93a7578/

Log:	Fix for ctypes.cast(bytes)

diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -143,7 +143,7 @@
         result = ptr
     elif isinstance(obj, bytes):
         result = tp()
-        result._buffer[0] = buffer(obj)._pypy_raw_address()
+        result._buffer[0] = memoryview(obj)._pypy_raw_address()
         return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"

From pypy.commits at gmail.com  Sat Sep 23 08:43:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 23 Sep 2017 05:43:59 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2628
Message-ID: <59c6570f.b5b8df0a.aec9c.0b1d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92447:167b802baf3b
Date: 2017-09-23 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/167b802baf3b/

Log:	Issue #2628

	Take the 'fake_frame' objects returned by sys._current_frames() on
	PyPy, and make inspect.isframe() return True on them. This lets a
	few more inspect functions work with them.

diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except AttributeError:
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 

From pypy.commits at gmail.com  Sun Sep 24 02:58:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 23 Sep 2017 23:58:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: Translation fix (uh,
 unsure what is going on here)
Message-ID: <59c75797.fb87df0a.58901.ee6a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92448:899e5245de1e
Date: 2017-09-24 08:58 +0200
http://bitbucket.org/pypy/pypy/changeset/899e5245de1e/

Log:	Translation fix (uh, unsure what is going on here)

diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -74,7 +74,7 @@
 # PyPy extension
 try:
     FakeFrameType = type(next(sys._current_frames().itervalues()))
-except AttributeError:
+except (AttributeError, StopIteration):
     FakeFrameType = FrameType
 
 SliceType = slice

From pypy.commits at gmail.com  Sun Sep 24 04:51:33 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 01:51:33 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release
Message-ID: <59c77215.e6361c0a.8267a.82c0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r92449:c2437cf9b7f1
Date: 2017-09-24 11:42 +0300
http://bitbucket.org/pypy/pypy/changeset/c2437cf9b7f1/

Log:	merge default into release

diff too long, truncating to 2000 out of 27632 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,6 @@
 syntax: glob
 *.py[co]
-*.sw[po]
+*.sw[pon]
 *~
 .*.swp
 .idea
@@ -8,6 +8,8 @@
 .pydevproject
 __pycache__
 
+.cache/
+.gdb_history
 syntax: regexp
 ^testresult$
 ^site-packages$
@@ -23,16 +25,17 @@
 ^pypy/module/cpyext/test/.+\.manifest$
 ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^pypy/module/cppyy/src/.+\.o$
-^pypy/module/cppyy/bench/.+\.so$
-^pypy/module/cppyy/bench/.+\.root$
-^pypy/module/cppyy/bench/.+\.d$
-^pypy/module/cppyy/src/.+\.errors$
-^pypy/module/cppyy/test/.+_rflx\.cpp$
-^pypy/module/cppyy/test/.+\.so$
-^pypy/module/cppyy/test/.+\.rootmap$
-^pypy/module/cppyy/test/.+\.exe$
-^pypy/module/cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/src/.+\.o$
+^pypy/module/_cppyy/bench/.+\.so$
+^pypy/module/_cppyy/bench/.+\.root$
+^pypy/module/_cppyy/bench/.+\.d$
+^pypy/module/_cppyy/src/.+\.errors$
+^pypy/module/_cppyy/test/.+_rflx\.cpp$
+^pypy/module/_cppyy/test/.+\.so$
+^pypy/module/_cppyy/test/.+\.rootmap$
+^pypy/module/_cppyy/test/.+\.exe$
+^pypy/module/_cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/.+/*\.pcm$
 ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
 ^pypy/doc/.+\.html$
 ^pypy/doc/config/.+\.rst$
@@ -85,8 +88,4 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
-^\.cache$
 
-pypy/module/cppyy/.+/*\.pcm
-
-
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -38,3 +38,5 @@
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
+c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
+a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@
 RUNINTERP = $(PYPY_EXECUTABLE)
 endif
 
-.PHONY: cffi_imports
+.PHONY: pypy-c cffi_imports
 
 pypy-c:
 	@echo
@@ -32,7 +32,7 @@
 	@echo "===================================================================="
 	@echo
 	@sleep 5
-	$(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+	cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit targetpypystandalone.py
 
 # Note: the -jN option, or MAKEFLAGS=-jN, are not usable.  They are
 # replaced with an opaque --jobserver option by the time this Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
+	PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -361,17 +361,20 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+            if _sys.maxint > 2 ** 32:
+                handle = int(handle)   # long -> int
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxint * 2 + 1))
-
+        return "<%s '%s', handle %x at %x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxint*2 + 1)),
+                id(self) & (_sys.maxint*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -218,6 +218,10 @@
 
         compiler.shared_lib_extension = so_ext
 
+def get_config_h_filename():
+    """Returns the path of pyconfig.h."""
+    inc_dir = get_python_inc(plat_specific=1)
+    return os.path.join(inc_dir, 'pyconfig.h')
 
 from sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
         return "-L" + dir
 
     def _is_gcc(self, compiler_name):
-        return "gcc" in compiler_name or "g++" in compiler_name
+        # XXX PyPy workaround, look at the big comment below for more
+        # context. On CPython, the hack below works fine because
+        # `compiler_name` contains the name of the actual compiler which was
+        # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+        # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+        # result is that we pass the wrong option to the compiler.
+        #
+        # The workaround is to *always* pretend to be GCC if we are on Linux:
+        # this should cover the vast majority of real systems, including the
+        # ones which use clang (which understands the '-Wl,-rpath' syntax as
+        # well)
+        return (sys.platform == "linux2" or
+                "gcc" in compiler_name or "g++" in compiler_name)
 
     def runtime_library_dir_option(self, dir):
         # XXX Hackish, at the very least.  See Python bug #445902:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/2.7/test/test_os.py
@@ -580,6 +580,7 @@
                  "getentropy() does not use a file descriptor")
 class URandomFDTests(unittest.TestCase):
     @unittest.skipUnless(resource, "test requires the resource module")
+    @test_support.impl_detail(pypy=False)    # on Linux, may use getrandom()
     def test_urandom_failure(self):
         # Check urandom() failing when it is not able to open /dev/random.
         # We spawn a new process to make the test more robust (if getrlimit()
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except (AttributeError, StopIteration):
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 
diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
 
     def __init__(self, message, category, filename, lineno, file=None,
                     line=None):
-        local_values = locals()
-        for attr in self._WARNING_DETAILS:
-            setattr(self, attr, local_values[attr])
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
         self._category_name = category.__name__ if category else None
 
     def __str__(self):
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -76,12 +76,16 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
-        # this seems to be a string if we're array of char, surprise!
-        from ctypes import c_char, c_wchar
-        if self._type_ is c_char:
-            return _rawffi.charp2string(resarray.buffer, self._length_)
-        if self._type_ is c_wchar:
-            return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+        from _rawffi.alt import types
+        # If a char_p or unichar_p is received, skip the string interpretation
+        if base._ffiargtype != types.Pointer(types.char_p) and \
+           base._ffiargtype != types.Pointer(types.unichar_p):
+            # this seems to be a string if we're array of char, surprise!
+            from ctypes import c_char, c_wchar
+            if self._type_ is c_char:
+                return _rawffi.charp2string(resarray.buffer, self._length_)
+            if self._type_ is c_wchar:
+                return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -82,7 +82,7 @@
         return False
 
     def in_dll(self, dll, name):
-        return self.from_address(dll._handle.getaddressindll(name))
+        return self.from_address(dll.__pypy_dll__.getaddressindll(name))
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -430,7 +430,7 @@
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
 
-        cdll = self.dll._handle
+        cdll = self.dll.__pypy_dll__
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffi_restype = restype.get_ffi_argtype()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -142,6 +142,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = buffer(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FFL) << 56) | \
+               ((value & 0x000000000000FF00L) << 40) | \
+               ((value & 0x0000000000FF0000L) << 24) | \
+               ((value & 0x00000000FF000000L) << 8) | \
+               ((value & 0x000000FF00000000L) >> 8) | \
+               ((value & 0x0000FF0000000000L) >> 24) | \
+               ((value & 0x00FF000000000000L) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -271,6 +319,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -40,6 +40,22 @@
         else:
             rawfields.append((f[0], f[1]._ffishape_))
 
+    # hack for duplicate field names
+    already_seen = set()
+    names1 = names
+    names = []
+    for f in names1:
+        if f not in already_seen:
+            names.append(f)
+            already_seen.add(f)
+    already_seen = set()
+    for i in reversed(range(len(rawfields))):
+        if rawfields[i][0] in already_seen:
+            rawfields[i] = (('$DUP%d$%s' % (i, rawfields[i][0]),)
+                            + rawfields[i][1:])
+        already_seen.add(rawfields[i][0])
+    # /hack
+
     _set_shape(self, rawfields, self._is_union)
 
     fields = {}
@@ -130,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -224,19 +241,27 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData):
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -8,6 +8,9 @@
 
 from _curses_cffi import ffi, lib
 
+version = b"2.2"
+__version__ = b"2.2"
+
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
 
@@ -60,10 +63,6 @@
 
 _setup()
 
-# Do we want this?
-# version = "2.2"
-# __version__ = "2.2"
-
 
 # ____________________________________________________________
 
@@ -913,101 +912,29 @@
     return None
 
 
-# XXX: Do something about the following?
-# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
-#  * and _curses.COLS */
-# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)
-# static int
-# update_lines_cols(void)
-# {
-#     PyObject *o;
-#     PyObject *m = PyImport_ImportModuleNoBlock("curses");
+# Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
+# and _curses.COLS
+def update_lines_cols():
+    globals()["LINES"] = lib.LINES
+    globals()["COLS"] = lib.COLS
+    try:
+        m = sys.modules["curses"]
+        m.LINES = lib.LINES
+        m.COLS = lib.COLS
+    except (KeyError, AttributeError):
+        pass
 
-#     if (!m)
-#         return 0;
 
-#     o = PyInt_FromLong(LINES);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     o = PyInt_FromLong(COLS);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     Py_DECREF(m);
-#     return 1;
-# }
-# #endif
+def resizeterm(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resizeterm(lines, columns), "resizeterm")
+    update_lines_cols()
 
-# #ifdef HAVE_CURSES_RESIZETERM
-# static PyObject *
-# PyCurses_ResizeTerm(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-#     PyObject *result;
 
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-
-# #endif
-
-# #ifdef HAVE_CURSES_RESIZE_TERM
-# static PyObject *
-# PyCurses_Resize_Term(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-
-#     PyObject *result;
-
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resize_term(lines, columns), "resize_term");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-# #endif /* HAVE_CURSES_RESIZE_TERM */
+def resize_term(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resize_term(lines, columns), "resize_term")
+    update_lines_cols()
 
 
 def setsyx(y, x):
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -87,6 +87,13 @@
 static const chtype A_CHARTEXT;
 static const chtype A_COLOR;
 
+static const chtype A_HORIZONTAL;
+static const chtype A_LEFT;
+static const chtype A_LOW;
+static const chtype A_RIGHT;
+static const chtype A_TOP;
+static const chtype A_VERTICAL;
+
 static const int BUTTON1_RELEASED;
 static const int BUTTON1_PRESSED;
 static const int BUTTON1_CLICKED;
@@ -202,6 +209,8 @@
 int resetty(void);
 int reset_prog_mode(void);
 int reset_shell_mode(void);
+int resizeterm(int, int);
+int resize_term(int, int);
 int savetty(void);
 int scroll(WINDOW *);
 int scrollok(WINDOW *, bool);
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -22,12 +22,27 @@
     linklibs = ['tcl', 'tk']
     libdirs = []
 else:
-    for _ver in ['', '8.6', '8.5', '']:
+    # On some Linux distributions, the tcl and tk libraries are
+    # stored in /usr/include, so we must check this case also
+    libdirs = []
+    found = False
+    for _ver in ['', '8.6', '8.5']:
         incdirs = ['/usr/include/tcl' + _ver]
         linklibs = ['tcl' + _ver, 'tk' + _ver]
-        libdirs = []
         if os.path.isdir(incdirs[0]):
+            found = True
             break
+    if not found:
+        for _ver in ['8.6', '8.5', '']:
+            incdirs = []
+            linklibs = ['tcl' + _ver, 'tk' + _ver]
+            if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
+                found = True
+                break
+    if not found:
+        sys.stderr.write("*** TCL libraries not found!  Falling back...\n")
+        incdirs = []
+        linklibs = ['tcl', 'tk']
 
 config_ffi = FFI()
 config_ffi.cdef("""
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -116,10 +116,20 @@
 
 @builtinify
 def dump(obj, file, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     Pickler(file, protocol).dump(obj)
 
 @builtinify
 def dumps(obj, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     file = StringIO()
     Pickler(file, protocol).dump(obj)
     return file.getvalue()
@@ -431,7 +441,14 @@
         self.append(obj)
 
     def find_class(self, module, name):
-        # Subclasses may override this
+        if self.find_global is None:
+            raise UnpicklingError(
+                "Global and instance pickles are not supported.")
+        return self.find_global(module, name)
+
+    def find_global(self, module, name):
+        # This can officially be patched directly in the Unpickler
+        # instance, according to the docs
         __import__(module)
         mod = sys.modules[module]
         klass = getattr(mod, name)
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.1
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.1"
-__version_info__ = (1, 10, 1)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -0,0 +1,145 @@
+#ifndef CFFI_MESSAGEBOX
+# ifdef _MSC_VER
+#  define CFFI_MESSAGEBOX  1
+# else
+#  define CFFI_MESSAGEBOX  0
+# endif
+#endif
+
+
+#if CFFI_MESSAGEBOX
+/* Windows only: logic to take the Python-CFFI embedding logic
+   initialization errors and display them in a background thread
+   with MessageBox.  The idea is that if the whole program closes
+   as a result of this problem, then likely it is already a console
+   program and you can read the stderr output in the console too.
+   If it is not a console program, then it will likely show its own
+   dialog to complain, or generally not abruptly close, and for this
+   case the background thread should stay alive.
+*/
+static void *volatile _cffi_bootstrap_text;
+
+static PyObject *_cffi_start_error_capture(void)
+{
+    PyObject *result = NULL;
+    PyObject *x, *m, *bi;
+
+    if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+            (void *)1, NULL) != NULL)
+        return (PyObject *)1;
+
+    m = PyImport_AddModule("_cffi_error_capture");
+    if (m == NULL)
+        goto error;
+
+    result = PyModule_GetDict(m);
+    if (result == NULL)
+        goto error;
+
+#if PY_MAJOR_VERSION >= 3
+    bi = PyImport_ImportModule("builtins");
+#else
+    bi = PyImport_ImportModule("__builtin__");
+#endif
+    if (bi == NULL)
+        goto error;
+    PyDict_SetItemString(result, "__builtins__", bi);
+    Py_DECREF(bi);
+
+    x = PyRun_String(
+        "import sys\n"
+        "class FileLike:\n"
+        "  def write(self, x):\n"
+        "    of.write(x)\n"
+        "    self.buf += x\n"
+        "fl = FileLike()\n"
+        "fl.buf = ''\n"
+        "of = sys.stderr\n"
+        "sys.stderr = fl\n"
+        "def done():\n"
+        "  sys.stderr = of\n"
+        "  return fl.buf\n",   /* make sure the returned value stays alive */
+        Py_file_input,
+        result, result);
+    Py_XDECREF(x);
+
+ error:
+    if (PyErr_Occurred())
+    {
+        PyErr_WriteUnraisable(Py_None);
+        PyErr_Clear();
+    }
+    return result;
+}
+
+#pragma comment(lib, "user32.lib")
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+#if PY_MAJOR_VERSION >= 3
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#else
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#endif
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+static void _cffi_stop_error_capture(PyObject *ecap)
+{
+    PyObject *s;
+    void *text;
+
+    if (ecap == (PyObject *)1)
+        return;
+
+    if (ecap == NULL)
+        goto error;
+
+    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
+    if (s == NULL)
+        goto error;
+
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+#if PY_MAJOR_VERSION >= 3
+    text = PyUnicode_AsWideCharString(s, NULL);
+#else
+    text = PyString_AsString(s);
+#endif
+
+    _cffi_bootstrap_text = text;
+
+    if (text != NULL)
+    {
+        HANDLE h;
+        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                         NULL, 0, NULL);
+        if (h != NULL)
+            CloseHandle(h);
+    }
+    /* decref the string, but it should stay alive as 'fl.buf'
+       in the small module above.  It will really be freed only if
+       we later get another similar error.  So it's a leak of at
+       most one copy of the small module.  That's fine for this
+       situation which is usually a "fatal error" anyway. */
+    Py_DECREF(s);
+    PyErr_Clear();
+    return;
+
+  error:
+    _cffi_bootstrap_text = NULL;
+    PyErr_Clear();
+}
+
+#else
+
+static PyObject *_cffi_start_error_capture(void) { return NULL; }
+static void _cffi_stop_error_capture(PyObject *ecap) { }
+
+#endif
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -95,6 +95,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
@@ -159,9 +160,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +175,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +220,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -1,7 +1,12 @@
 
 /***** Support code for embedding *****/
 
-#if defined(_MSC_VER)
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#if defined(_WIN32)
 #  define CFFI_DLLEXPORT  __declspec(dllexport)
 #elif defined(__GNUC__)
 #  define CFFI_DLLEXPORT  __attribute__((visibility("default")))
@@ -109,6 +114,8 @@
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
+#include "_cffi_errors.h"
+
 
 #define _cffi_call_python_org  _cffi_exports[_CFFI_CPIDX]
 
@@ -220,8 +227,16 @@
         /* Print as much information as potentially useful.
            Debugging load-time failures with embedding is not fun
         */
+        PyObject *ecap;
         PyObject *exception, *v, *tb, *f, *modules, *mod;
         PyErr_Fetch(&exception, &v, &tb);
+        ecap = _cffi_start_error_capture();
+        f = PySys_GetObject((char *)"stderr");
+        if (f != NULL && f != Py_None) {
+            PyFile_WriteString(
+                "Failed to initialize the Python-CFFI embedding logic:\n\n", f);
+        }
+
         if (exception != NULL) {
             PyErr_NormalizeException(&exception, &v, &tb);
             PyErr_Display(exception, v, tb);
@@ -230,10 +245,9 @@
         Py_XDECREF(v);
         Py_XDECREF(tb);
 
-        f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -249,6 +263,7 @@
             PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
             PyFile_WriteString("\n\n", f);
         }
+        _cffi_stop_error_capture(ecap);
     }
     result = -1;
     goto done;
@@ -515,3 +530,7 @@
 #undef cffi_compare_and_swap
 #undef cffi_write_barrier
 #undef cffi_read_barrier
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -75,9 +75,10 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
+        self._typecache = model.get_typecache(backend)
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
-        for name in backend.__dict__:
+        for name in list(backend.__dict__):
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
@@ -393,12 +394,17 @@
             replace_with = ' ' + replace_with
         return self._backend.getcname(cdecl, replace_with)
 
-    def gc(self, cdata, destructor):
+    def gc(self, cdata, destructor, size=0):
         """Return a new cdata object that points to the same
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
+
+        The optional 'size' gives an estimate of the size, used to
+        trigger the garbage collection more eagerly.  So far only used
+        on PyPy.  It tells the GC that the returned object keeps alive
+        roughly 'size' bytes of external memory.
         """
-        return self._backend.gcp(cdata, destructor)
+        return self._backend.gcp(cdata, destructor, size)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
@@ -764,7 +770,7 @@
         if sys.platform != "win32":
             return backend.load_library(None, flags)
         name = "c"    # Windows: load_library(None) fails, but this works
-                      # (backward compatibility hack only)
+                      # on Python 2 (backward compatibility hack only)
     first_error = None
     if '.' in name or '/' in name or os.sep in name:
         try:
@@ -774,6 +780,9 @@
     import ctypes.util
     path = ctypes.util.find_library(name)
     if path is None:
+        if name == "c" and sys.platform == "win32" and sys.version_info >= (3,):
+            raise OSError("dlopen(None) cannot work on Windows for Python 3 "
+                          "(see http://bugs.python.org/issue23606)")
         msg = ("ctypes.util.find_library() did not manage "
                "to locate a library called %r" % (name,))
         if first_error is not None:
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
 
     _weakref_cache_ref = None
 
-    def gcp(self, cdata, destructor):
+    def gcp(self, cdata, destructor, size=0):
         if self._weakref_cache_ref is None:
             import weakref
             class MyRef(weakref.ref):
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,12 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-_NUM_PRIM          = 48
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,8 +132,12 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,9 +117,13 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
@@ -163,6 +168,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
@@ -561,22 +568,26 @@
 
 
 global_lock = allocate_lock()
+_typecache_cffi_backend = weakref.WeakValueDictionary()
+
+def get_typecache(backend):
+    # returns _typecache_cffi_backend if backend is the _cffi_backend
+    # module, or type(backend).__typecache if backend is an instance of
+    # CTypesBackend (or some FakeBackend class during tests)
+    if isinstance(backend, types.ModuleType):
+        return _typecache_cffi_backend
+    with global_lock:
+        if not hasattr(type(backend), '__typecache'):
+            type(backend).__typecache = weakref.WeakValueDictionary()
+        return type(backend).__typecache
 
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
     try:
-        return ffi._backend.__typecache[key]
+        return ffi._typecache[key]
     except KeyError:
         pass
-    except AttributeError:
-        # initialize the __typecache attribute, either at the module level
-        # if ffi._backend is a module, or at the class level if ffi._backend
-        # is some instance.
-        if isinstance(ffi._backend, types.ModuleType):
-            ffi._backend.__typecache = weakref.WeakValueDictionary()
-        else:
-            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
@@ -584,7 +595,7 @@
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
-    cache = ffi._backend.__typecache
+    cache = ffi._typecache
     with global_lock:
         res1 = cache.get(key)
         if res1 is None:
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,12 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -303,10 +308,10 @@
                 base_module_name,))
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
+            i = lines.index('#include "_cffi_errors.h"\n')
+            lines[i:i+1] = self._rel_readlines('_cffi_errors.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,8 +410,11 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
+        prnt('#if PY_MAJOR_VERSION >= 3')
+        prnt('    return NULL;')
+        prnt('#endif')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
         # 'export_symbols', so instead of fighting it, just give up and
@@ -423,21 +431,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +485,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -506,7 +516,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -515,8 +525,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -524,8 +537,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -566,12 +581,15 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.BasePrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+            elif tp.name != 'long double' and not tp.is_complex_type():
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
@@ -734,21 +752,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1203,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -296,7 +296,7 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.PrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
@@ -869,6 +872,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -268,12 +268,22 @@
         assert abs(d) == 1
         source = getcurrent()
         source.tempval = arg
-        if d > 0:
-            cando = self.balance < 0
-            dir = d
-        else:
-            cando = self.balance > 0
-            dir = 0
+        while True:
+            if d > 0:
+                cando = self.balance < 0
+                dir = d
+            else:
+                cando = self.balance > 0
+                dir = 0
+
+            if cando and self.queue[0]._tasklet_killed:
+                # issue #2595: the tasklet was killed while waiting.
+                # drop that tasklet from consideration and try again.
+                self.balance += d
+                self.queue.popleft()
+            else:
+                # normal path
+                break
 
         if _channel_callback is not None:
             _channel_callback(self, source, dir, not cando)
@@ -348,6 +358,8 @@
     module.
     """
     tempval = None
+    _tasklet_killed = False
+
     def __new__(cls, func=None, label=''):
         res = coroutine.__new__(cls)
         res.label = label
@@ -395,6 +407,7 @@
         If the exception passes the toplevel frame of the tasklet,
         the tasklet will silently die.
         """
+        self._tasklet_killed = True
         if not self.is_zombie:
             # Killing the tasklet by throwing TaskletExit exception.
             coroutine.kill(self)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson", "_jitlog"
+    "_csv", "_cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
@@ -67,10 +67,12 @@
         if name in translation_modules:
             translation_modules.remove(name)
 
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # not tested on win32
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # not tested on win32
     if "faulthandler" in working_modules:
         working_modules.remove("faulthandler")  # missing details
+    if "_vmprof" in working_modules:
+        working_modules.remove("_vmprof")  # FIXME: missing details
 
     # The _locale module is needed by site.py on Windows
     default_modules.add("_locale")
@@ -79,8 +81,8 @@
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # depends on ctypes
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # depends on ctypes
 
 #if sys.platform.startswith("linux"):
 #    _mach = os.popen('uname -m', 'r').read().strip()
@@ -92,7 +94,7 @@
     '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
-    'cppyy': [('objspace.usemodules.cpyext', True)],
+    '_cppyy': [('objspace.usemodules.cpyext', True)],
     'faulthandler': [('objspace.usemodules._vmprof', True)],
     }
 module_suggests = {
@@ -224,11 +226,6 @@
                    "use specialised tuples",
                    default=False),
 
-        BoolOption("withcelldict",
-                   "use dictionaries that are optimized for being used as module dicts",
-                   default=False,
-                   requires=[("objspace.honor__builtins__", False)]),
-
         BoolOption("withliststrategies",
                    "enable optimized ways to store lists of primitives ",
                    default=True),
@@ -288,7 +285,7 @@
 
     # extra optimizations with the JIT
     if level == 'jit':
-        config.objspace.std.suggest(withcelldict=True)
+        pass # none at the moment
 
 
 def enable_allworkingmodules(config):
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -10,6 +10,18 @@
 minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
 of memory on a 32-bit machine and 4GB on a 64-bit machine.
 
+Before you start
+----------------
+
+Our normal development workflow avoids a full translation by using test-driven
+development. You can read more about how to develop PyPy here_, and latest
+translated (hopefully functional) binary packages are available on our
+buildbot's `nightly builds`_
+
+.. _here: getting-started-dev.html
+.. _`nightly builds`: http://buildbot.pypy.org/nightly
+
+You will need the build dependencies below to run the tests.
 
 Clone the repository
 --------------------
@@ -93,7 +105,8 @@
     libsqlite3
 
 curses
-    libncurses
+    libncurses-dev   (for PyPy2)
+    libncursesw-dev  (for PyPy3)
 
 gdbm
     libgdbm-dev
@@ -106,12 +119,20 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian, this is the command to install all build-time dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
     tk-dev libgc-dev python-cffi \
-    liblzma-dev  # For lzma on PyPy3.
+    liblzma-dev libncursesw-dev      # these two only needed on PyPy3
 
 On Fedora::
 
@@ -138,22 +159,61 @@
 Run the translation
 -------------------
 
+We usually translate in the ``pypy/goal`` directory, so all the following
+commands assume your ``$pwd`` is there.
+
 Translate with JIT::
 
-    cd pypy/goal
     pypy ../../rpython/bin/rpython --opt=jit
 
 Translate without JIT::
 
-    cd pypy/goal
     pypy ../../rpython/bin/rpython --opt=2
 
+Note this translates pypy via the ``targetpypystandalone.py`` file, so these
+are shorthand for::
+
+    pypy ../../rpython/bin/rpython <rpython options> targetpypystandalone.py <pypy options>
+
+More help is availabe via ``--help`` at either option position, and more info
+can be found in the :doc:`config/index` section.
+
 (You can use ``python`` instead of ``pypy`` here, which will take longer
 but works too.)
 
-If everything works correctly this will create an executable ``pypy-c`` in the
-current directory. The executable behaves mostly like a normal Python
-interpreter (see :doc:`cpython_differences`).
+If everything works correctly this will:
+
+1. Run the rpython `translation chain`_, producing a database of the
+   entire pypy interpreter. This step is currently singe threaded, and RAM
+   hungry. As part of this step,  the chain creates a large number of C code
+   files and a Makefile to compile them in a
+   directory controlled by the ``PYPY_USESSION_DIR`` environment variable.  
+2. Create an executable ``pypy-c`` by running the Makefile. This step can
+   utilize all possible cores on the machine.  
+3. Copy the needed binaries to the current directory.  
+4. Generate c-extension modules for any cffi-based stdlib modules.  
+
+
+The resulting executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`), and is ready for testing, for
+use as a base interpreter for a new virtualenv, or for packaging into a binary
+suitable for installation on another machine running the same OS as the build
+machine. 
+
+Note that step 4 is merely done as a convenience, any of the steps may be rerun
+without rerunning the previous steps.
+
+.. _`translation chain`: https://rpython.readthedocs.io/en/latest/translation.html
+
+
+Making a debug build of PyPy
+----------------------------
+
+If the Makefile is rerun with the lldebug or lldebug0 target, appropriate
+compilation flags are added to add debug info and reduce compiler optimizations
+to ``-O0`` respectively. If you stop in a debugger, you will see the
+very wordy machine-generated C code from the rpython translation step, which
+takes a little bit of reading to relate back to the rpython code.
 
 Build cffi import libraries for the stdlib
 ------------------------------------------
@@ -167,14 +227,6 @@
 
 .. _`out-of-line API mode`: http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line
 
-Translating with non-standard options
--------------------------------------
-
-It is possible to have non-standard features enabled for translation,
-but they are not really tested any more.  Look, for example, at the
-:doc:`objspace proxies <objspace-proxies>` document.
-
-
 Packaging (preparing for installation)
 --------------------------------------
 
@@ -195,6 +247,31 @@
 ``/tmp/usession-YOURNAME/build/``.  You can then either move the file
 hierarchy or unpack the ``.tar.bz2`` at the correct place.
 
+It is recommended to use package.py because custom scripts will
+invariably become out-of-date.  If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation.  If you install PyPy as root without
+pre-compiling them, normal users will get errors:
+
+* PyPy 2.5.1 or earlier: normal users would see permission errors.
+  Installers need to run ``pypy -c "import gdbm"`` and other similar
+  commands at install time; the exact list is in 
+  :source:`pypy/tool/release/package.py <package.py>`.  Users
+  seeing a broken installation of PyPy can fix it after-the-fact if they
+  have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
+
+* PyPy 2.6 and later: anyone would get ``ImportError: no module named
+  _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
+  ``lib_pypy`` directory during the installation process (plus others;
+  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  Users seeing a broken
+  installation of PyPy can fix it after-the-fact, by running ``pypy
+  /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
+  called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
+  module for PyPy.  You can move it at any place where modules are
+  normally found: e.g. in your project's main directory, or in a
+  directory that you add to the env var ``PYTHONPATH``.
+
 
 Installation
 ------------
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.4'
+version = '5.8'
 # The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/config/objspace.std.withcelldict.txt b/pypy/doc/config/objspace.std.withcelldict.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.withcelldict.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Enable cell-dicts. This optimization is not helpful without the JIT. In the
-presence of the JIT, it greatly helps looking up globals.
diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst
--- a/pypy/doc/configuration.rst
+++ b/pypy/doc/configuration.rst
@@ -188,4 +188,6 @@
 can be found on the ``config`` attribute of all ``TranslationContext``
 instances and are described in :source:`rpython/config/translationoption.py`. The interpreter options
 are attached to the object space, also under the name ``config`` and are
-described in :source:`pypy/config/pypyoption.py`.
+described in :source:`pypy/config/pypyoption.py`. Both set of options are
+documented in the :doc:`config/index` section.
+
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
deleted file mode 100644
--- a/pypy/doc/cppyy.rst
+++ /dev/null
@@ -1,672 +0,0 @@
-cppyy: C++ bindings for PyPy
-============================
-
-The cppyy module delivers dynamic Python-C++ bindings.
-It is designed for automation, high performance, scale, interactivity, and
-handling all of modern C++ (11, 14, etc.).
-It is based on `Cling`_ which, through `LLVM`_/`clang`_, provides C++
-reflection and interactivity.
-Reflection information is extracted from C++ header files.
-Cppyy itself is built into PyPy (an alternative exists for CPython), but
-it requires a `backend`_, installable through pip, to interface with Cling.
-
-.. _Cling: https://root.cern.ch/cling
-.. _LLVM: http://llvm.org/
-.. _clang: http://clang.llvm.org/
-.. _backend: https://pypi.python.org/pypi/PyPy-cppyy-backend
-
-
-Installation
-------------
-
-This assumes PyPy2.7 v5.7 or later; earlier versions use a Reflex-based cppyy
-module, which is no longer supported.
-Both the tooling and user-facing Python codes are very backwards compatible,
-however.
-Further dependencies are cmake (for general build), Python2.7 (for LLVM), and
-a modern C++ compiler (one that supports at least C++11).
-
-Assuming you have a recent enough version of PyPy installed, use pip to
-complete the installation of cppyy::
-
- $ MAKE_NPROCS=4 pypy-c -m pip install --verbose PyPy-cppyy-backend
-
-Set the number of parallel builds ('4' in this example, through the MAKE_NPROCS
-environment variable) to a number appropriate for your machine.
-The building process may take quite some time as it includes a customized
-version of LLVM as part of Cling, which is why --verbose is recommended so that
-you can see the build progress.
-
-The default installation will be under
-$PYTHONHOME/site-packages/cppyy_backend/lib,
-which needs to be added to your dynamic loader path (LD_LIBRARY_PATH).
-If you need the dictionary and class map generation tools (used in the examples
-below), you need to add $PYTHONHOME/site-packages/cppyy_backend/bin to your
-executable path (PATH).
-
-
-Basic bindings example
-----------------------
-
-These examples assume that cppyy_backend is pointed to by the environment
-variable CPPYYHOME, and that CPPYYHOME/lib is added to LD_LIBRARY_PATH and
-CPPYYHOME/bin to PATH.
-
-Let's first test with a trivial example whether all packages are properly
-installed and functional.
-Create a C++ header file with some class in it (all functions are made inline
-for convenience; if you have out-of-line code, link with it as appropriate)::
-
-    $ cat MyClass.h
-    class MyClass {
-    public:
-        MyClass(int i = -99) : m_myint(i) {}
-
-        int GetMyInt() { return m_myint; }
-        void SetMyInt(int i) { m_myint = i; }
-
-    public:
-        int m_myint;
-    };
-
-Then, generate the bindings using ``genreflex`` (installed under
-cppyy_backend/bin in site_packages), and compile the code::
-
-    $ genreflex MyClass.h
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$CPPYYHOME/lib -lCling
-
-Next, make sure that the library can be found through the dynamic lookup path

From pypy.commits at gmail.com  Sun Sep 24 04:51:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 01:51:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: update version, contributors
Message-ID: <59c77217.50b51c0a.297c4.92d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92450:085ebf62ad46
Date: 2017-09-24 11:49 +0300
http://bitbucket.org/pypy/pypy/changeset/085ebf62ad46/

Log:	update version, contributors

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -27,8 +27,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -69,6 +69,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -77,14 +78,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -123,6 +123,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -149,11 +150,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -161,6 +164,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -171,6 +175,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -181,14 +186,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -210,7 +213,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -242,6 +244,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -272,6 +275,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -292,12 +296,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"2.7.13"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.9.0-alpha0"
-#define PYPY_VERSION_NUM  0x05090000
+#define PYPY_VERSION "5.10.0-alpha0"
+#define PYPY_VERSION_NUM  0x05100000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 9, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 10, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Sun Sep 24 05:02:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 02:02:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: update version to 5.10 alpha
Message-ID: <59c774ac.37b9df0a.fd4da.d3f7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92451:be41e3ac0a29
Date: 2017-09-24 11:58 +0300
http://bitbucket.org/pypy/pypy/changeset/be41e3ac0a29/

Log:	update version to 5.10 alpha

diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -29,8 +29,8 @@
 #define PY_VERSION		"3.5.3"
 
 /* PyPy version as a string */
-#define PYPY_VERSION "5.9.0-alpha0"
-#define PYPY_VERSION_NUM  0x05090000
+#define PYPY_VERSION "5.10.0-alpha0"
+#define PYPY_VERSION_NUM  0x05100000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py
--- a/pypy/module/sys/version.py
+++ b/pypy/module/sys/version.py
@@ -10,7 +10,7 @@
 #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py
 CPYTHON_API_VERSION        = 1013   #XXX # sync with include/modsupport.h
 
-PYPY_VERSION               = (5, 9, 0, "alpha", 0)    #XXX # sync patchlevel.h
+PYPY_VERSION               = (5, 10, 0, "alpha", 0)    #XXX # sync patchlevel.h
 
 
 import pypy

From pypy.commits at gmail.com  Sun Sep 24 05:02:41 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 02:02:41 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release
Message-ID: <59c774b1.d28bdf0a.4d390.c085@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r92452:631824fbda33
Date: 2017-09-24 12:00 +0300
http://bitbucket.org/pypy/pypy/changeset/631824fbda33/

Log:	merge py3.5 into release

diff too long, truncating to 2000 out of 31743 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,6 @@
 syntax: glob
 *.py[co]
-*.sw[po]
+*.sw[pon]
 *~
 .*.swp
 .idea
@@ -10,6 +10,8 @@
 .venv
 .cache
 
+.cache/
+.gdb_history
 syntax: regexp
 ^testresult$
 ^site-packages$
@@ -25,16 +27,17 @@
 ^pypy/module/cpyext/test/.+\.manifest$
 ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^pypy/module/cppyy/src/.+\.o$
-^pypy/module/cppyy/bench/.+\.so$
-^pypy/module/cppyy/bench/.+\.root$
-^pypy/module/cppyy/bench/.+\.d$
-^pypy/module/cppyy/src/.+\.errors$
-^pypy/module/cppyy/test/.+_rflx\.cpp$
-^pypy/module/cppyy/test/.+\.so$
-^pypy/module/cppyy/test/.+\.rootmap$
-^pypy/module/cppyy/test/.+\.exe$
-^pypy/module/cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/src/.+\.o$
+^pypy/module/_cppyy/bench/.+\.so$
+^pypy/module/_cppyy/bench/.+\.root$
+^pypy/module/_cppyy/bench/.+\.d$
+^pypy/module/_cppyy/src/.+\.errors$
+^pypy/module/_cppyy/test/.+_rflx\.cpp$
+^pypy/module/_cppyy/test/.+\.so$
+^pypy/module/_cppyy/test/.+\.rootmap$
+^pypy/module/_cppyy/test/.+\.exe$
+^pypy/module/_cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/.+/*\.pcm$
 ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
 ^pypy/doc/.+\.html$
 ^pypy/doc/config/.+\.rst$
@@ -90,8 +93,4 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
-^\.cache$
 
-pypy/module/cppyy/.+/*\.pcm
-
-
diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -38,3 +38,5 @@
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
+c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
+a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@
 RUNINTERP = $(PYPY_EXECUTABLE)
 endif
 
-.PHONY: cffi_imports
+.PHONY: pypy-c cffi_imports
 
 pypy-c:
 	@echo
@@ -32,7 +32,7 @@
 	@echo "===================================================================="
 	@echo
 	@sleep 5
-	$(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+	cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit targetpypystandalone.py
 
 # Note: the -jN option, or MAKEFLAGS=-jN, are not usable.  They are
 # replaced with an opaque --jobserver option by the time this Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
+	PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
--- a/extra_tests/test_decimal.py
+++ b/extra_tests/test_decimal.py
@@ -1,3 +1,6 @@
+import pytest
+from hypothesis import example, settings, given, strategies as st
+
 import pickle
 import sys
 
@@ -8,52 +11,112 @@
 # import _decimal as C
 # import _pydecimal as P
 
+ at pytest.yield_fixture(params=[C, P], ids=['_decimal', '_pydecimal'])
+def module(request):
+    yield request.param
 
-class TestPythonAPI:
+# Translate symbols.
+CondMap = {
+        C.Clamped:             P.Clamped,
+        C.ConversionSyntax:    P.ConversionSyntax,
+        C.DivisionByZero:      P.DivisionByZero,
+        C.DivisionImpossible:  P.InvalidOperation,
+        C.DivisionUndefined:   P.DivisionUndefined,
+        C.Inexact:             P.Inexact,
+        C.InvalidContext:      P.InvalidContext,
+        C.InvalidOperation:    P.InvalidOperation,
+        C.Overflow:            P.Overflow,
+        C.Rounded:             P.Rounded,
+        C.Subnormal:           P.Subnormal,
+        C.Underflow:           P.Underflow,
+        C.FloatOperation:      P.FloatOperation,
+}
 
-    def check_equal(self, val, proto):
-        d = C.Decimal(val)
-        p = pickle.dumps(d, proto)
-        assert d == pickle.loads(p)
+def check_same_flags(flags_C, flags_P):
+    for signal in flags_C:
+        assert flags_C[signal] == flags_P[CondMap[signal]]
 
-    def test_C(self):
+
+def test_C():
+    sys.modules["decimal"] = C
+    import decimal
+    d = decimal.Decimal('1')
+    assert isinstance(d, C.Decimal)
+    assert isinstance(d, decimal.Decimal)
+    assert isinstance(d.as_tuple(), C.DecimalTuple)
+
+    assert d == C.Decimal('1')
+
+def check_round_trip(val, proto):
+    d = C.Decimal(val)
+    p = pickle.dumps(d, proto)
+    assert d == pickle.loads(p)
+
+def test_pickle():
+    v = '-3.123e81723'
+    for proto in range(pickle.HIGHEST_PROTOCOL + 1):
         sys.modules["decimal"] = C
-        import decimal
-        d = decimal.Decimal('1')
-        assert isinstance(d, C.Decimal)
-        assert isinstance(d, decimal.Decimal)
-        assert isinstance(d.as_tuple(), C.DecimalTuple)
+        check_round_trip('-3.141590000', proto)
+        check_round_trip(v, proto)
 
-        assert d == C.Decimal('1')
+        cd = C.Decimal(v)
+        pd = P.Decimal(v)
+        cdt = cd.as_tuple()
+        pdt = pd.as_tuple()
+        assert cdt.__module__ == pdt.__module__
 
-    def test_pickle(self):
-        v = '-3.123e81723'
-        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
-            sys.modules["decimal"] = C
-            self.check_equal('-3.141590000', proto)
-            self.check_equal(v, proto)
+        p = pickle.dumps(cdt, proto)
+        r = pickle.loads(p)
+        assert isinstance(r, C.DecimalTuple)
+        assert cdt == r
 
-            cd = C.Decimal(v)
-            pd = P.Decimal(v)
-            cdt = cd.as_tuple()
-            pdt = pd.as_tuple()
-            assert cdt.__module__ == pdt.__module__
+        sys.modules["decimal"] = C
+        p = pickle.dumps(cd, proto)
+        sys.modules["decimal"] = P
+        r = pickle.loads(p)
+        assert isinstance(r, P.Decimal)
+        assert r == pd
 
-            p = pickle.dumps(cdt, proto)
-            r = pickle.loads(p)
-            assert isinstance(r, C.DecimalTuple)
-            assert cdt == r
+        sys.modules["decimal"] = C
+        p = pickle.dumps(cdt, proto)
+        sys.modules["decimal"] = P
+        r = pickle.loads(p)
+        assert isinstance(r, P.DecimalTuple)
+        assert r == pdt
 
-            sys.modules["decimal"] = C
-            p = pickle.dumps(cd, proto)
-            sys.modules["decimal"] = P
-            r = pickle.loads(p)
-            assert isinstance(r, P.Decimal)
-            assert r == pd
+def test_compare_total(module):
+    assert module.Decimal('12').compare_total(module.Decimal('12.0')) == 1
+    assert module.Decimal('4367').compare_total(module.Decimal('NaN')) == -1
 
-            sys.modules["decimal"] = C
-            p = pickle.dumps(cdt, proto)
-            sys.modules["decimal"] = P
-            r = pickle.loads(p)
-            assert isinstance(r, P.DecimalTuple)
-            assert r == pdt
+def test_compare_total_mag(module):
+    assert module.Decimal(1).compare_total_mag(-2) == -1
+
+def convert_arg(module, arg):
+    if isinstance(arg, module.Decimal):
+        return arg
+    elif type(arg).__name__ == 'Decimal':
+        return module.Decimal(str(arg))
+    else:
+        return arg
+
+from fractions import Fraction
+from decimal import Decimal
+
+ at given(st.decimals(), st.decimals() | st.fractions())
+def test_lt(d1, d2):
+    with C.localcontext(C.ExtendedContext) as ctx_C:
+        d1_C = convert_arg(C, d1)
+        d2_C = convert_arg(C, d2)
+        try:
+            res_C = d1_C < d2_C
+        except Exception as e:
+            res_C = str(type(e))
+    with P.localcontext(P.ExtendedContext) as ctx_P:
+        d1_P = convert_arg(P, d1)
+        d2_P = convert_arg(P, d2)
+        try:
+            res_P = d1_P < d2_P
+        except Exception as e:
+            res_P = str(type(e))
+    assert res_C == res_P
+    check_same_flags(ctx_C.flags, ctx_P.flags)
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -361,17 +361,20 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+            if _sys.maxint > 2 ** 32:
+                handle = int(handle)   # long -> int
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxint * 2 + 1))
-
+        return "<%s '%s', handle %x at %x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxint*2 + 1)),
+                id(self) & (_sys.maxint*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -218,6 +218,10 @@
 
         compiler.shared_lib_extension = so_ext
 
+def get_config_h_filename():
+    """Returns the path of pyconfig.h."""
+    inc_dir = get_python_inc(plat_specific=1)
+    return os.path.join(inc_dir, 'pyconfig.h')
 
 from sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
         return "-L" + dir
 
     def _is_gcc(self, compiler_name):
-        return "gcc" in compiler_name or "g++" in compiler_name
+        # XXX PyPy workaround, look at the big comment below for more
+        # context. On CPython, the hack below works fine because
+        # `compiler_name` contains the name of the actual compiler which was
+        # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+        # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+        # result is that we pass the wrong option to the compiler.
+        #
+        # The workaround is to *always* pretend to be GCC if we are on Linux:
+        # this should cover the vast majority of real systems, including the
+        # ones which use clang (which understands the '-Wl,-rpath' syntax as
+        # well)
+        return (sys.platform == "linux2" or
+                "gcc" in compiler_name or "g++" in compiler_name)
 
     def runtime_library_dir_option(self, dir):
         # XXX Hackish, at the very least.  See Python bug #445902:
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/lib-python/2.7/test/test_os.py b/lib-python/2.7/test/test_os.py
--- a/lib-python/2.7/test/test_os.py
+++ b/lib-python/2.7/test/test_os.py
@@ -580,6 +580,7 @@
                  "getentropy() does not use a file descriptor")
 class URandomFDTests(unittest.TestCase):
     @unittest.skipUnless(resource, "test requires the resource module")
+    @test_support.impl_detail(pypy=False)    # on Linux, may use getrandom()
     def test_urandom_failure(self):
         # Check urandom() failing when it is not able to open /dev/random.
         # We spawn a new process to make the test more robust (if getrlimit()
diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
 
     def __init__(self, message, category, filename, lineno, file=None,
                     line=None):
-        local_values = locals()
-        for attr in self._WARNING_DETAILS:
-            setattr(self, attr, local_values[attr])
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
         self._category_name = category.__name__ if category else None
 
     def __str__(self):
diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -346,16 +346,18 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxsize * 2 + 1))
+        return "<%s '%s', handle %x at 0x%x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxsize*2 + 1)),
+                id(self) & (_sys.maxsize*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/3/ctypes/test/test_byteswap.py b/lib-python/3/ctypes/test/test_byteswap.py
--- a/lib-python/3/ctypes/test/test_byteswap.py
+++ b/lib-python/3/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
 from binascii import hexlify
 
 from ctypes import *
-from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).decode().upper()
@@ -43,7 +42,6 @@
         with self.assertRaises(AttributeError):
             little.z = 24
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -71,7 +69,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -100,7 +97,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -129,7 +125,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -148,7 +143,6 @@
         self.assertAlmostEqual(s.value, math.pi, places=6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -176,7 +170,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -212,7 +205,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -241,7 +233,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -265,7 +256,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -810,7 +810,8 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        # PyPy fix: returns type(self)() instead of date()
+        return type(self)(year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1285,7 +1286,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return time(hour, minute, second, microsecond, tzinfo)
+        # PyPy fix: returns type(self)() instead of time()
+        return type(self)(hour, minute, second, microsecond, tzinfo)
 
     # Pickle support.
 
@@ -1497,7 +1499,8 @@
             microsecond = self.microsecond
         if tzinfo is True:
             tzinfo = self.tzinfo
-        return datetime(year, month, day, hour, minute, second, microsecond,
+        # PyPy fix: returns type(self)() instead of datetime()
+        return type(self)(year, month, day, hour, minute, second, microsecond,
                         tzinfo)
 
     def astimezone(self, tz=None):
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -73,7 +73,7 @@
     g['CCSHARED'] = "-fPIC"
     g['LDSHARED'] = "cc -pthread -shared"
     g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = so_ext
+    g['SHLIB_SUFFIX'] = ".so"
     g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"
@@ -81,6 +81,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py
--- a/lib-python/3/stat.py
+++ b/lib-python/3/stat.py
@@ -139,13 +139,21 @@
 def filemode(mode):
     """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
     perm = []
+
+    # The first group gets a question mark if none of the bits match the mode.
+    empty = "?"
+
     for table in _filemode_table:
         for bit, char in table:
             if mode & bit == bit:
                 perm.append(char)
                 break
         else:
-            perm.append("-")
+            perm.append(empty)
+
+        # All the rest of the positions get a - if the bits don't match.
+        empty = "-"
+
     return "".join(perm)
 
 
diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -1588,9 +1588,15 @@
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEADDR))
         if reuseport_supported:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            try:
+                self.assertFalse(
+                    sock.getsockopt(
+                        socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            except OSError:
+                # Python's socket module was compiled using modern headers
+                # thus defining SO_REUSEPORT but this process is running
+                # under an older kernel that does not support SO_REUSEPORT.
+                reuseport_supported = False
         self.assertFalse(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))
diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -1663,7 +1663,8 @@
         self.assertEqual(b.foo, 3)
         self.assertEqual(b.__class__, D)
 
-    @unittest.expectedFailure
+    #@unittest.expectedFailure --- on CPython.  On PyPy, the test passes
+    @support.impl_detail(cpython=False)
     def test_bad_new(self):
         self.assertRaises(TypeError, object.__new__)
         self.assertRaises(TypeError, object.__new__, '')
diff --git a/lib-python/3/test/test_importlib/builtin/test_loader.py b/lib-python/3/test/test_importlib/builtin/test_loader.py
--- a/lib-python/3/test/test_importlib/builtin/test_loader.py
+++ b/lib-python/3/test/test_importlib/builtin/test_loader.py
@@ -1,6 +1,8 @@
 from .. import abc
 from .. import util
 
+from importlib.machinery import BuiltinImporter
+
 machinery = util.import_importlib('importlib.machinery')
 
 import sys
@@ -14,7 +16,7 @@
 
     def setUp(self):
         self.verification = {'__name__': 'errno', '__package__': '',
-                             '__loader__': self.machinery.BuiltinImporter}
+                             '__loader__': BuiltinImporter}  # PyPy change
 
     def verify(self, module):
         """Verify that the module matches against what it should have."""
diff --git a/lib-python/3/test/test_importlib/extension/test_loader.py b/lib-python/3/test/test_importlib/extension/test_loader.py
--- a/lib-python/3/test/test_importlib/extension/test_loader.py
+++ b/lib-python/3/test/test_importlib/extension/test_loader.py
@@ -88,6 +88,7 @@
 
     def setUp(self):
         self.name = '_testmultiphase'
+        __import__(self.name)  # PyPy hack
         finder = self.machinery.FileFinder(None)
         self.spec = importlib.util.find_spec(self.name)
         assert self.spec
@@ -145,7 +146,8 @@
             importlib.reload(module)
             self.assertIs(ex_class, module.Example)
 
-    def test_try_registration(self):
+    # XXX: PyPy doesn't support the PyState_* functions yet
+    def XXXtest_try_registration(self):
         '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work'''
         module = self.load_module()
         with self.subTest('PyState_FindModule'):
diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -271,6 +271,11 @@
                 if n is not None and n > 4:
                     n += 10**6
                 return n
+            def read(self, n):   # PyPy calls read(), not readinto()
+                result = super().read(n)
+                if len(result) > 4:
+                    result += b'\x00' * (10**6)
+                return result
         for value in (1.0, 1j, b'0123456789', '0123456789'):
             self.assertRaises(ValueError, marshal.load,
                               BadReader(marshal.dumps(value)))
@@ -348,7 +353,8 @@
     strobj = "abcde"*3
     dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
 
-    def helper3(self, rsample, recursive=False, simple=False):
+    def helper3(self, rsample, recursive=False, simple=False,
+                check_sharing=True, check_non_sharing=True):
         #we have two instances
         sample = (rsample, rsample)
 
@@ -358,28 +364,35 @@
         n3 = CollectObjectIDs(set(), marshal.loads(s3))
 
         #same number of instances generated
-        self.assertEqual(n3, n0)
+        # except in one corner case on top of pypy, for code objects
+        if check_sharing:
+            self.assertEqual(n3, n0)
 
         if not recursive:
             #can compare with version 2
             s2 = marshal.dumps(sample, 2)
             n2 = CollectObjectIDs(set(), marshal.loads(s2))
             #old format generated more instances
-            self.assertGreater(n2, n0)
+            # except on pypy where equal ints or floats always have
+            # the same id anyway
+            if check_non_sharing:
+                self.assertGreater(n2, n0)
 
             #if complex objects are in there, old format is larger
-            if not simple:
+            if check_non_sharing and not simple:
                 self.assertGreater(len(s2), len(s3))
             else:
                 self.assertGreaterEqual(len(s2), len(s3))
 
     def testInt(self):
         self.helper(self.intobj)
-        self.helper3(self.intobj, simple=True)
+        self.helper3(self.intobj, simple=True,
+                     check_non_sharing=support.check_impl_detail())
 
     def testFloat(self):
         self.helper(self.floatobj)
-        self.helper3(self.floatobj)
+        self.helper3(self.floatobj,
+                     check_non_sharing=support.check_impl_detail())
 
     def testStr(self):
         self.helper(self.strobj)
@@ -395,7 +408,7 @@
         if __file__.endswith(".py"):
             code = compile(code, __file__, "exec")
         self.helper(code)
-        self.helper3(code)
+        self.helper3(code, check_sharing=support.check_impl_detail())
 
     def testRecursion(self):
         d = dict(self.dictobj)
diff --git a/lib-python/3/test/test_pyexpat.py b/lib-python/3/test/test_pyexpat.py
--- a/lib-python/3/test/test_pyexpat.py
+++ b/lib-python/3/test/test_pyexpat.py
@@ -11,7 +11,7 @@
 from xml.parsers import expat
 from xml.parsers.expat import errors
 
-from test.support import sortdict
+from test.support import sortdict, impl_detail
 
 
 class SetAttributeTest(unittest.TestCase):
@@ -446,6 +446,7 @@
         self.assertEqual(os.path.basename(entry[0]), filename)
         self.assertEqual(entry[2], funcname)
 
+    @impl_detail("PyPy does not have pyexpat.c", pypy=False)
     def test_exception(self):
         parser = expat.ParserCreate()
         parser.StartElementHandler = self.StartElementHandler
diff --git a/lib-python/3/test/test_stat.py b/lib-python/3/test/test_stat.py
--- a/lib-python/3/test/test_stat.py
+++ b/lib-python/3/test/test_stat.py
@@ -138,6 +138,10 @@
             self.assertS_IS("REG", st_mode)
             self.assertEqual(modestr, '-r--r--r--')
             self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444)
+
+            # If there are only permission bits, no type bytes, a question
+            # mark is rendered in the type field.
+            self.assertEqual(self.statmod.filemode(0o420), '?r---w----')
         else:
             os.chmod(TESTFN, 0o700)
             st_mode, modestr = self.get_mode()
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -397,9 +397,16 @@
             self.assertTrue('linux' in suffix, suffix)
         if re.match('(i[3-6]86|x86_64)$', machine):
             if ctypes.sizeof(ctypes.c_char_p()) == 4:
-                self.assertTrue(suffix.endswith('i386-linux-gnu.so') \
-                                or suffix.endswith('x86_64-linux-gnux32.so'),
-                                suffix)
+                self.assertTrue(
+                    suffix.endswith((
+                        'i386-linux-gnu.so',
+                        'i486-linux-gnu.so',
+                        'i586-linux-gnu.so',
+                        'i686-linux-gnu.so',
+                        'x86_64-linux-gnux32.so',
+                    )),
+                    suffix,
+                )
             else: # 8 byte pointer size
                 self.assertTrue(suffix.endswith('x86_64-linux-gnu.so'), suffix)
 
diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -5,8 +5,15 @@
 it renames the compiled shared object to _pypy_openssl.so (which means
 that cryptography can ship their own cffi backend)
 
-NOTE: currently, we have changed ``_cffi_src/openssl/callbacks.py`` to
-not rely on the CPython C API.
+NOTE: currently, we have the following changes:
+
+* ``_cffi_src/openssl/callbacks.py`` to not rely on the CPython C API
+  (this change is now backported)
+
+* ``_cffi_src/utils.py`` for issue #2575 (29c9a89359e4)
+
+* ``_cffi_src/openssl/x509_vfy.py`` for issue #2605 (ca4d0c90f5a1)
+
 
 # Tests?
 
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
deleted file mode 100644
index 180c02ff82d3363f34a334aae22c9876d4c96481..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py
@@ -221,10 +221,16 @@
 static const long X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM = 0;
 static const long X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED = 0;
 static const long X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 = 0;
+#ifndef X509_V_ERR_HOSTNAME_MISMATCH
 static const long X509_V_ERR_HOSTNAME_MISMATCH = 0;
+#endif
+#ifndef X509_V_ERR_EMAIL_MISMATCH
 static const long X509_V_ERR_EMAIL_MISMATCH = 0;
+#endif
+#ifndef X509_V_ERR_IP_ADDRESS_MISMATCH
 static const long X509_V_ERR_IP_ADDRESS_MISMATCH = 0;
 #endif
+#endif
 
 /* OpenSSL 1.0.2beta2+ verification parameters */
 #if CRYPTOGRAPHY_OPENSSL_102BETA2_OR_GREATER && \
diff --git a/lib_pypy/_cffi_ssl/_cffi_src/utils.py b/lib_pypy/_cffi_ssl/_cffi_src/utils.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/utils.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/utils.py
@@ -47,9 +47,19 @@
     # is legal, but the following will fail to compile:
     #   int foo(int);
     #   int foo(short);
+    #
+    # XXX <arigo> No, it is a bad idea.  OpenSSL itself tends to tweak
+    # the definitions, like adding a 'const' (see issue #2575).  Every
+    # time they do so, it makes a gratuitous break in this code.  It is
+    # better to rely on the C compiler for that, which is a little bit
+    # more flexible.  That's the point of set_source().  We can still
+    # re-enable the line ``#functions +`` below to get the original
+    # behavior.  (I would enable it during tests, but I don't find any
+    # custom test at all..??)
+    #
     verify_source = "\n".join(
         includes +
-        functions +
+        #functions +
         customizations
     )
     ffi = build_ffi(
diff --git a/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn b/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn
deleted file mode 100644
index 40344f6cee5cb001b73dd3a9a203015568831391..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -173,14 +173,13 @@
 
     return tuple(dn)
 
-STATIC_BIO_BUF = ffi.new("char[]", 2048)
-
 def _bio_get_str(biobuf):
-    length = lib.BIO_gets(biobuf, STATIC_BIO_BUF, len(STATIC_BIO_BUF)-1)
+    bio_buf = ffi.new("char[]", 2048)
+    length = lib.BIO_gets(biobuf, bio_buf, len(bio_buf)-1)
     if length < 0:
         if biobuf: lib.BIO_free(biobuf)
         raise ssl_error(None)
-    return _str_with_len(STATIC_BIO_BUF, length)
+    return _str_with_len(bio_buf, length)
 
 def _decode_certificate(certificate):
     retval = {}
diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,4 +1,5 @@
 import sys
+import os
 import traceback
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -100,18 +101,17 @@
             errval = SSL_ERROR_WANT_CONNECT
         elif err == SSL_ERROR_SYSCALL:
             if e == 0:
-                if ret == 0 or obj.socket is not None:
+                if ret == 0 or obj.socket is None:
                     errtype = SSLEOFError
                     errstr = "EOF occurred in violation of protocol"
                     errval = SSL_ERROR_EOF
                 elif ret == -1 and obj.socket is not None:
                     # the underlying BIO reported an I/0 error
                     lib.ERR_clear_error()
-                    s = obj.get_socket_or_None()
-                    s.errorhandler()
-                    assert 0, "must not get here"
-                    #errno = ffi.errno
-                    #return IOError(errno)
+                    # s = obj.get_socket_or_None()
+                    # XXX: Windows?
+                    errno = ffi.errno
+                    return OSError(errno, os.strerror(errno))
                 else:
                     errtype = SSLSyscallError
                     errstr = "Some I/O error occurred"
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -74,12 +74,16 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
-        # this seems to be a string if we're array of char, surprise!
-        from ctypes import c_char, c_wchar
-        if self._type_ is c_char:
-            return _rawffi.charp2string(resarray.buffer, self._length_)
-        if self._type_ is c_wchar:
-            return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+        from _rawffi.alt import types
+        # If a char_p or unichar_p is received, skip the string interpretation
+        if base._ffiargtype != types.Pointer(types.char_p) and \
+           base._ffiargtype != types.Pointer(types.unichar_p):
+            # this seems to be a string if we're array of char, surprise!
+            from ctypes import c_char, c_wchar
+            if self._type_ is c_char:
+                return _rawffi.charp2string(resarray.buffer, self._length_)
+            if self._type_ is c_wchar:
+                return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -82,7 +82,7 @@
         return False
 
     def in_dll(self, dll, name):
-        return self.from_address(dll._handle.getaddressindll(name))
+        return self.from_address(dll.__pypy_dll__.getaddressindll(name))
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -430,7 +430,7 @@
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
 
-        cdll = self.dll._handle
+        cdll = self.dll.__pypy_dll__
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffi_restype = restype.get_ffi_argtype()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -141,6 +141,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = memoryview(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FF) << 56) | \
+               ((value & 0x000000000000FF00) << 40) | \
+               ((value & 0x0000000000FF0000) << 24) | \
+               ((value & 0x00000000FF000000) << 8) | \
+               ((value & 0x000000FF00000000) >> 8) | \
+               ((value & 0x0000FF0000000000) >> 24) | \
+               ((value & 0x00FF000000000000) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -265,6 +313,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -40,6 +40,22 @@
         else:
             rawfields.append((f[0], f[1]._ffishape_))
 
+    # hack for duplicate field names
+    already_seen = set()
+    names1 = names
+    names = []
+    for f in names1:
+        if f not in already_seen:
+            names.append(f)
+            already_seen.add(f)
+    already_seen = set()
+    for i in reversed(range(len(rawfields))):
+        if rawfields[i][0] in already_seen:
+            rawfields[i] = (('$DUP%d$%s' % (i, rawfields[i][0]),)
+                            + rawfields[i][1:])
+        already_seen.add(rawfields[i][0])
+    # /hack
+
     _set_shape(self, rawfields, self._is_union)
 
     fields = {}
@@ -130,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -224,18 +241,26 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData, metaclass=StructOrUnionMeta):
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -8,6 +8,9 @@
 
 from _curses_cffi import ffi, lib
 
+version = b"2.2"
+__version__ = b"2.2"
+
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
 
@@ -60,10 +63,6 @@
 
 _setup()
 
-# Do we want this?
-# version = "2.2"
-# __version__ = "2.2"
-
 
 # ____________________________________________________________
 
@@ -404,6 +403,17 @@
             raise error("getch requires 0 or 2 arguments")
         return val
 
+    def get_wch(self, *args):
+        wch = ffi.new("int[1]")
+        if len(args) == 0:
+            val = lib.wget_wch(self._win, wch)
+        elif len(args) == 2:
+            val = lib.mvwget_wch(self._win, *args, wch)
+        else:
+            raise error("get_wch requires 0 or 2 arguments")
+        _check_ERR(val, "get_wch")
+        return wch[0]
+
     def getkey(self, *args):
         if len(args) == 0:
             val = lib.wgetch(self._win)
@@ -919,101 +929,29 @@
     return None
 
 
-# XXX: Do something about the following?
-# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
-#  * and _curses.COLS */
-# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)
-# static int
-# update_lines_cols(void)
-# {
-#     PyObject *o;
-#     PyObject *m = PyImport_ImportModuleNoBlock("curses");
+# Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
+# and _curses.COLS
+def update_lines_cols():
+    globals()["LINES"] = lib.LINES
+    globals()["COLS"] = lib.COLS
+    try:
+        m = sys.modules["curses"]
+        m.LINES = lib.LINES
+        m.COLS = lib.COLS
+    except (KeyError, AttributeError):
+        pass
 
-#     if (!m)
-#         return 0;
 
-#     o = PyInt_FromLong(LINES);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     o = PyInt_FromLong(COLS);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     Py_DECREF(m);
-#     return 1;
-# }
-# #endif
+def resizeterm(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resizeterm(lines, columns), "resizeterm")
+    update_lines_cols()
 
-# #ifdef HAVE_CURSES_RESIZETERM
-# static PyObject *
-# PyCurses_ResizeTerm(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-#     PyObject *result;
 
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-
-# #endif
-
-# #ifdef HAVE_CURSES_RESIZE_TERM
-# static PyObject *
-# PyCurses_Resize_Term(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-
-#     PyObject *result;
-
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resize_term(lines, columns), "resize_term");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-# #endif /* HAVE_CURSES_RESIZE_TERM */
+def resize_term(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resize_term(lines, columns), "resize_term")
+    update_lines_cols()
 
 
 def setsyx(y, x):
@@ -1078,6 +1016,11 @@
     return _check_ERR(lib.ungetch(_chtype(ch)), "ungetch")
 
 
+def unget_wch(ch):
+    _ensure_initialised()
+    return _check_ERR(lib.unget_wch(_chtype(ch)), "unget_wch")
+
+
 def use_env(flag):
     lib.use_env(flag)
     return None
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,3 +1,4 @@
+import os
 from cffi import FFI, VerificationError
 
 
@@ -17,6 +18,11 @@
     # error message
     raise e_last
 
+def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncursesw'):
+        return ['/usr/include/ncursesw']
+    return []
+
 
 ffi = FFI()
 
@@ -59,7 +65,8 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=[find_curses_library(), 'panel'])
+""", libraries=[find_curses_library(), 'panel'],
+     include_dirs=find_curses_include_dirs())
 
 
 ffi.cdef("""
@@ -70,6 +77,8 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
+typedef int... wint_t;
+
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -105,6 +114,13 @@
 static const chtype A_CHARTEXT;
 static const chtype A_COLOR;
 
+static const chtype A_HORIZONTAL;
+static const chtype A_LEFT;
+static const chtype A_LOW;
+static const chtype A_RIGHT;
+static const chtype A_TOP;
+static const chtype A_VERTICAL;
+
 static const int BUTTON1_RELEASED;
 static const int BUTTON1_PRESSED;
 static const int BUTTON1_CLICKED;
@@ -160,6 +176,8 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -220,6 +238,8 @@
 int resetty(void);
 int reset_prog_mode(void);
 int reset_shell_mode(void);
+int resizeterm(int, int);
+int resize_term(int, int);
 int savetty(void);
 int scroll(WINDOW *);
 int scrollok(WINDOW *, bool);
@@ -233,6 +253,7 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
+int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -489,13 +489,16 @@
                     vv.exp = 0
                     multiplied = Decimal._new_empty()
                     denom = Decimal(other.denominator)
-                    with _CatchStatus(context) as (ctx, status_ptr):
-                        _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd,
-                                        ctx, status_ptr)
-                    multiplied._mpd.exp += exp  # XXX probably a bug
-                                                # in _decimal.c
+                    maxctx = _ffi.new("struct mpd_context_t*")
+                    _mpdec.mpd_maxcontext(maxctx)
+                    status_ptr = _ffi.new("uint32_t*")
+                    _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd,
+                                    maxctx, status_ptr)
+                    multiplied._mpd.exp = exp
                 finally:
                     _mpdec.mpd_del(vv)
+                if status_ptr[0] != 0:
+                    raise ValueError("exact conversion for comparison failed")
 
                 return multiplied, numerator
             else:
@@ -719,8 +722,8 @@
 
     compare = _make_binary_operation('compare')
     compare_signal = _make_binary_operation('compare_signal')
-    compare_total = _make_binary_operation('compare')
-    compare_total_mag = _make_binary_operation('compare')
+    compare_total = _make_binary_operation('compare_total')
+    compare_total_mag = _make_binary_operation('compare_total_mag')
     logical_and = _make_binary_operation('logical_and')
     logical_or = _make_binary_operation('logical_or')
     logical_xor = _make_binary_operation('logical_xor')
diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -10,6 +10,7 @@
 import weakref
 import sys
 import io
+import __pypy__
 
 from _lzma_cffi import ffi, lib as m
 
@@ -63,6 +64,10 @@
     m._pylzma_stream_init(ret)
     return ffi.gc(ret, m.lzma_end)
 
+def _release_lzma_stream(st):
+    ffi.gc(st, None)
+    m.lzma_end(st)
+
 def add_constant(c):
     globals()[c] = getattr(m, 'LZMA_' + c)
 
@@ -148,39 +153,39 @@
 def parse_filter_spec_lzma(id, preset=m.LZMA_PRESET_DEFAULT, **kwargs):
     ret = ffi.new('lzma_options_lzma*')
     if m.lzma_lzma_preset(ret, preset):
-        raise LZMAError("Invalid...")
+        raise LZMAError("Invalid compression preset: %s" % preset)
     for arg, val in kwargs.items():
         if arg in ('dict_size', 'lc', 'lp', 'pb', 'nice_len', 'depth'):
             setattr(ret, arg, val)
         elif arg in ('mf', 'mode'):
             setattr(ret, arg, int(val))
         else:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for LZMA filter")
     return ret
 
 def parse_filter_spec(spec):
     if not isinstance(spec, collections.Mapping):
-        raise TypeError("Filter...")
+        raise TypeError("Filter specifier must be a dict or dict-like object")
     ret = ffi.new('lzma_filter*')
     try:
         ret.id = spec['id']
     except KeyError:
-        raise ValueError("Filter...")
+        raise ValueError("Filter specifier must have an \"id\" entry")
     if ret.id in (m.LZMA_FILTER_LZMA1, m.LZMA_FILTER_LZMA2):
         try:
             options = parse_filter_spec_lzma(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for LZMA filter")
     elif ret.id == m.LZMA_FILTER_DELTA:
         try:
             options = parse_filter_spec_delta(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for delta filter")
     elif ret.id in BCJ_FILTERS:
         try:
             options = parse_filter_spec_bcj(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for BCJ filter")
     else:
         raise ValueError("Invalid %d" % (ret.id,))
 
@@ -204,7 +209,9 @@
 
 def parse_filter_chain_spec(filterspecs):
     if len(filterspecs) > m.LZMA_FILTERS_MAX:
-        raise ValueError("Too...")
+        raise ValueError(
+            "Too many filters - liblzma supports a maximum of %s" %
+            m.LZMA_FILTERS_MAX)
     filters = ffi.new('lzma_filter[]', m.LZMA_FILTERS_MAX+1)
     _owns[filters] = children = []
     for i in range(m.LZMA_FILTERS_MAX+1):
@@ -236,7 +243,7 @@
     elif filter.id in BCJ_FILTERS:
         add_opts('lzma_options_bcj', 'start_offset')
     else:
-        raise ValueError("Invalid...")
+        raise ValueError("Invalid filter ID: %s" % filter.id)
     return spec
 
 def _decode_filter_properties(filter_id, encoded_props):
@@ -420,25 +427,26 @@
 
     For one-shot decompression, use the decompress() function instead.
     """
-    def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None, header=None, check=None, unpadded_size=None):
+    def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None,
+                 header=None, check=None, unpadded_size=None):
         decoder_flags = m.LZMA_TELL_ANY_CHECK | m.LZMA_TELL_NO_CHECK
-        #decoder_flags = 0
         if memlimit is not None:
             if format == FORMAT_RAW:
-                raise ValueError("Cannot sp...")
-            #memlimit = long(memlimit)
+                raise ValueError("Cannot specify memory limit with FORMAT_RAW")
         else:
             memlimit = m.UINT64_MAX
 
         if format == FORMAT_RAW and filters is None:
-            raise ValueError("Must...")
+            raise ValueError("Must specify filters for FORMAT_RAW")
         elif format != FORMAT_RAW and filters is not None:
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify filters except with FORMAT_RAW")
 
         if format == FORMAT_BLOCK and (header is None or unpadded_size is None or check is None):
-            raise ValueError("Must...")
+            raise ValueError("Must specify header, unpadded_size and check "
+                             "with FORMAT_BLOCK")
         elif format != FORMAT_BLOCK and (header is not None or unpadded_size is not None or check is not None):
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify header, unpadded_size or check "
+                             "except with FORMAT_BLOCK")
 
         format = _parse_format(format)
         self.lock = threading.Lock()
@@ -476,7 +484,7 @@
             self.expected_size = block.compressed_size
             catch_lzma_error(m.lzma_block_decoder, self.lzs, block)
         else:
-            raise ValueError("invalid...")
+            raise ValueError("invalid container format: %s" % format)
 
     def pre_decompress_left_data(self, buf, buf_size):
         # in this case there is data left that needs to be processed before the first
@@ -551,7 +559,7 @@
             raise TypeError("max_length parameter object cannot be interpreted as an integer")
         with self.lock:
             if self.eof:
-                raise EOFError("Already...")
+                raise EOFError("Already at end of stream")
             lzs = self.lzs
             data = to_bytes(data)
             buf = ffi.new('uint8_t[]', data)
@@ -648,6 +656,16 @@
         raise TypeError("cannot serialize '%s' object" %
                         self.__class__.__name__)
 
+
+# Issue #2579: Setting up the stream for encoding takes around 17MB of
+# RAM on my Linux 64 system.  So we call add_memory_pressure(17MB) when
+# we create the stream.  In flush(), we actively free the stream even
+# though we could just leave it to the GC (but 17MB is too much for
+# doing that sanely); at this point we call add_memory_pressure(-17MB)
+# to cancel the original increase.
+COMPRESSION_STREAM_SIZE = 1024*1024*17
+
+
 class LZMACompressor(object):
     """
     LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)
@@ -679,15 +697,16 @@
     """
     def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None):
         if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE):
-            raise ValueError("Integrity...")
+            raise ValueError("Integrity checks are only supported by FORMAT_XZ")
         if preset is not None and filters is not None:
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify both preset and filter chain")
         if preset is None:
             preset = m.LZMA_PRESET_DEFAULT
         format = _parse_format(format)
         self.lock = threading.Lock()
         self.flushed = 0
         self.lzs = _new_lzma_stream()
+        __pypy__.add_memory_pressure(COMPRESSION_STREAM_SIZE)
         if format == FORMAT_XZ:
             if filters is None:
                 if check == -1:
@@ -702,19 +721,19 @@
             if filters is None:
                 options = ffi.new('lzma_options_lzma*')
                 if m.lzma_lzma_preset(options, preset):
-                    raise LZMAError("Invalid...")
+                    raise LZMAError("Invalid compression preset: %s" % preset)
                 catch_lzma_error(m.lzma_alone_encoder, self.lzs,
                     options)
             else:
                 raise NotImplementedError
         elif format == FORMAT_RAW:
             if filters is None:
-                raise ValueError("Must...")
+                raise ValueError("Must specify filters for FORMAT_RAW")
             filters = parse_filter_chain_spec(filters)
             catch_lzma_error(m.lzma_raw_encoder, self.lzs,
                 filters)
         else:
-            raise ValueError("Invalid...")
+            raise ValueError("invalid container format: %s" % format)
 
     def compress(self, data):
         """
@@ -728,7 +747,7 @@
         """
         with self.lock:
             if self.flushed:
-                raise ValueError("Compressor...")
+                raise ValueError("Compressor has been flushed")
             return self._compress(data)
 
     def _compress(self, data, action=m.LZMA_RUN):
@@ -769,9 +788,12 @@
     def flush(self):
         with self.lock:
             if self.flushed:
-                raise ValueError("Repeated...")
+                raise ValueError("Repeated call to flush()")
             self.flushed = 1
-            return self._compress(b'', action=m.LZMA_FINISH)
+            result = self._compress(b'', action=m.LZMA_FINISH)
+            __pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE)
+            _release_lzma_stream(self.lzs)
+        return result
 
     def __getstate__(self):
         raise TypeError("cannot serialize '%s' object" %
diff --git a/lib_pypy/_testmultiphase.c b/lib_pypy/_testmultiphase.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_testmultiphase.c
@@ -0,0 +1,627 @@
+/* Copied from CPython's Modules/_testmultiphase.c */
+/***************************************************/
+
+/* Testing module for multi-phase initialization of extension modules (PEP 489)
+ */
+
+#include "Python.h"
+
+/* Example objects */
+typedef struct {
+    PyObject_HEAD
+    PyObject            *x_attr;        /* Attributes dictionary */
+} ExampleObject;
+
+/* Example methods */
+
+static int
+Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->x_attr);
+    return 0;
+}
+
+static int
+Example_finalize(ExampleObject *self)
+{
+    Py_CLEAR(self->x_attr);
+    return 0;
+}
+
+static PyObject *
+Example_demo(ExampleObject *self, PyObject *args)
+{
+    PyObject *o = NULL;
+    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+        return NULL;
+    if (o != NULL && PyUnicode_Check(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyMethodDef Example_methods[] = {
+    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+        PyDoc_STR("demo() -> None")},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static PyObject *
+Example_getattro(ExampleObject *self, PyObject *name)
+{
+    if (self->x_attr != NULL) {
+        PyObject *v = PyDict_GetItem(self->x_attr, name);
+        if (v != NULL) {
+            Py_INCREF(v);
+            return v;
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)self, name);
+}
+
+static int
+Example_setattr(ExampleObject *self, char *name, PyObject *v)
+{
+    if (self->x_attr == NULL) {
+        self->x_attr = PyDict_New();
+        if (self->x_attr == NULL)
+            return -1;
+    }
+    if (v == NULL) {
+        int rv = PyDict_DelItemString(self->x_attr, name);
+        if (rv < 0)
+            PyErr_SetString(PyExc_AttributeError,
+                "delete non-existing Example attribute");
+        return rv;
+    }
+    else
+        return PyDict_SetItemString(self->x_attr, name, v);
+}
+
+static PyType_Slot Example_Type_slots[] = {
+    {Py_tp_doc, "The Example type"},
+//    {Py_tp_finalize, Example_finalize},
+    {Py_tp_traverse, Example_traverse},
+    {Py_tp_getattro, Example_getattro},
+    {Py_tp_setattr, Example_setattr},
+    {Py_tp_methods, Example_methods},
+    {0, 0},
+};
+
+static PyType_Spec Example_Type_spec = {
+    "_testimportexec.Example",
+    sizeof(ExampleObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+    Example_Type_slots
+};
+
+/* Function of two integers returning integer */
+
+PyDoc_STRVAR(testexport_foo_doc,
+"foo(i,j)\n\
+\n\
+Return the sum of i and j.");
+
+static PyObject *
+testexport_foo(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j))
+        return NULL;
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+/* Test that PyState registration fails  */
+
+//PyDoc_STRVAR(call_state_registration_func_doc,
+//"register_state(0): call PyState_FindModule()\n\
+//register_state(1): call PyState_AddModule()\n\
+//register_state(2): call PyState_RemoveModule()");
+//
+//static PyObject *
+//call_state_registration_func(PyObject *mod, PyObject *args)
+//{
+//    int i, ret;
+//    PyModuleDef *def = PyModule_GetDef(mod);
+//    if (def == NULL) {
+//        return NULL;
+//    }
+//    if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i))
+//        return NULL;
+//    switch (i) {
+//        case 0:
+//            mod = PyState_FindModule(def);
+//            if (mod == NULL) {
+//                Py_RETURN_NONE;
+//            }
+//            return mod;
+//        case 1:
+//            ret = PyState_AddModule(mod, def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//        case 2:
+//            ret = PyState_RemoveModule(def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//    }
+//    Py_RETURN_NONE;
+//}
+
+
+static PyType_Slot Str_Type_slots[] = {
+    {Py_tp_base, NULL}, /* filled out in module exec function */
+    {0, 0},
+};
+
+static PyType_Spec Str_Type_spec = {
+    "_testimportexec.Str",
+    0,
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Str_Type_slots
+};
+
+static PyMethodDef testexport_methods[] = {
+    {"foo",             testexport_foo,         METH_VARARGS,
+        testexport_foo_doc},
+//    {"call_state_registration_func",  call_state_registration_func,
+//        METH_VARARGS, call_state_registration_func_doc},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static int execfunc(PyObject *m)
+{
+    PyObject *temp = NULL;
+
+    /* Due to cross platform compiler issues the slots must be filled
+     * here. It's required for portability to Windows without requiring
+     * C++. */
+    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+
+    /* Add a custom type */
+    temp = PyType_FromSpec(&Example_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Example", temp) != 0)
+        goto fail;
+
+    /* Add an exception type */
+    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "error", temp) != 0)
+        goto fail;
+
+    /* Add Str */
+    temp = PyType_FromSpec(&Str_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Str", temp) != 0)
+        goto fail;
+
+    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+        goto fail;
+
+    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+        goto fail;
+
+    return 0;
+ fail:
+    return -1;
+}
+
+/* Helper for module definitions; there'll be a lot of them */
+#define TEST_MODULE_DEF(name, slots, methods) { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */ \
+    name,                                       /* m_name */ \
+    PyDoc_STR("Test module " name),             /* m_doc */ \
+    0,                                          /* m_size */ \
+    methods,                                    /* m_methods */ \
+    slots,                                      /* m_slots */ \
+    NULL,                                       /* m_traverse */ \
+    NULL,                                       /* m_clear */ \
+    NULL,                                       /* m_free */ \
+}
+
+PyModuleDef_Slot main_slots[] = {
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};

From pypy.commits at gmail.com  Sun Sep 24 14:27:11 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 24 Sep 2017 11:27:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement PyGenObject so that C code can
 recognize @types.coroutine old-style generator-coroutines
Message-ID: <59c7f8ff.a7a6df0a.e4867.2049@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92453:ce24631719e2
Date: 2017-09-24 19:27 +0100
http://bitbucket.org/pypy/pypy/changeset/ce24631719e2/

Log:	Implement PyGenObject so that C code can recognize @types.coroutine
	old-style generator-coroutines

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -12,7 +12,8 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
-    CO_GENERATOR, CO_COROUTINE, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
+    CO_GENERATOR, CO_COROUTINE, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY,
+    CO_ITERABLE_COROUTINE)
 from pypy.tool import dis3
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -18,6 +18,8 @@
     CO_VARKEYWORDS = 0x0008,
     CO_NESTED      = 0x0010,
     CO_GENERATOR   = 0x0020,
+    CO_COROUTINE=0x0080,
+    CO_ITERABLE_COROUTINE=0x0100,
 )
 ALL_CODE_FLAGS = unrolling_iterable(CODE_FLAGS.items())
 
diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
--- a/pypy/module/cpyext/genobject.py
+++ b/pypy/module/cpyext/genobject.py
@@ -1,7 +1,29 @@
 from pypy.interpreter.generator import GeneratorIterator, Coroutine
-from pypy.module.cpyext.api import build_type_checkers
+from pypy.module.cpyext.api import (
+    build_type_checkers, cts, parse_dir, bootstrap_function)
+from pypy.module.cpyext.pyobject import make_typedescr, as_pyobj
+from pypy.module.cpyext.object import _dealloc
+
+cts.parse_header(parse_dir / 'cpyext_genobject.h')
+
+ at bootstrap_function
+def init_genobject(space):
+    make_typedescr(GeneratorIterator.typedef,
+                   basestruct=cts.gettype('PyGenObject'),
+                   attach=gi_attach,
+                   dealloc=_dealloc)
 
 
 PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
 
 _, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
+
+def gi_attach(space, py_obj, w_obj, w_userdata=None):
+    py_obj.c_gi_code = as_pyobj(space, w_obj.pycode)
+
+def gi_realize(space, py_obj):
+    raise NotImplementedError(
+        "PyPy doesn't support creation of generators from the C-API.")
+
+def gi_dealloc(space, py_obj):
+    _dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/code.h b/pypy/module/cpyext/include/code.h
--- a/pypy/module/cpyext/include/code.h
+++ b/pypy/module/cpyext/include/code.h
@@ -20,6 +20,11 @@
 #define CO_VARKEYWORDS  0x0008
 #define CO_NESTED       0x0010
 #define CO_GENERATOR    0x0020
+  
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+   ``async def`` keywords) */
+#define CO_COROUTINE            0x0080
+#define CO_ITERABLE_COROUTINE   0x0100
 
 #define CO_FUTURE_DIVISION         0x02000
 #define CO_FUTURE_ABSOLUTE_IMPORT  0x04000
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
--- a/pypy/module/cpyext/test/test_genobject.py
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -28,7 +28,28 @@
         assert PyCoro_CheckExact(space, w_coroutine)
 
 class AppTestCoroutine(AppTestCpythonExtensionBase):
-    def test_simple(self):
+    def test_generator_coroutine(self):
+        module = self.import_extension('test_gen', [
+            ('is_coroutine', 'METH_O',
+             '''
+             if (!PyGen_CheckExact(args))
+                Py_RETURN_NONE;
+             PyObject* co = ((PyGenObject*)args)->gi_code;
+             if (((PyCodeObject*)co)->co_flags & CO_ITERABLE_COROUTINE)
+                Py_RETURN_TRUE;
+             else
+                Py_RETURN_FALSE;
+             ''')])
+        def it():
+            yield 42
+
+        print(module.is_coroutine(it()))
+        assert module.is_coroutine(it()) is False
+        from types import coroutine
+        assert module.is_coroutine(coroutine(it)()) is True
+
+
+    def test_await(self):
         """
         module = self.import_extension('test_coroutine', [
             ('await_', 'METH_O',

From pypy.commits at gmail.com  Sun Sep 24 14:29:06 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 24 Sep 2017 11:29:06 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add missing files
Message-ID: <59c7f972.4cda1c0a.9bdf9.8b94@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92454:3aa19dff16dd
Date: 2017-09-24 19:29 +0100
http://bitbucket.org/pypy/pypy/changeset/3aa19dff16dd/

Log:	Add missing files

diff --git a/pypy/module/cpyext/include/genobject.h b/pypy/module/cpyext/include/genobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/genobject.h
@@ -0,0 +1,12 @@
+#ifndef Py_GENOBJECT_H
+#define Py_GENOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "cpyext_genobject.h"
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GENOBJECT_H */
diff --git a/pypy/module/cpyext/parse/cpyext_genobject.h b/pypy/module/cpyext/parse/cpyext_genobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_genobject.h
@@ -0,0 +1,4 @@
+typedef struct {
+    PyObject_HEAD
+    PyObject* gi_code;
+} PyGenObject;

From pypy.commits at gmail.com  Sun Sep 24 14:30:29 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 11:30:29 -0700 (PDT)
Subject: [pypy-commit] pypy default: start release cycle
Message-ID: <59c7f9c5.c1151c0a.f9fe0.eda0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92455:689624ab0c14
Date: 2017-09-23 23:54 +0300
http://bitbucket.org/pypy/pypy/changeset/689624ab0c14/

Log:	start release cycle

diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.9.0.rst
    release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.9.0.rst
@@ -0,0 +1,131 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.9 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 includes the upstream stdlib version 3.5.3.
+
+XXXX hilights here
+
+
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy and PEP 489 was fully implemented. Of course the
+bug fixes and performance enhancements mentioned above are part of both PyPy
+2.7 and PyPy 3.5.
+
+CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.11,
+improving an already great package for interfacing with C.
+
+As always, this release fixed many other issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.9 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _vmprof: http://vmprof.readthedocs.io
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017)
+======================================================================================
+
+See also issues that were resolved_
+
+Note that these are also merged into PyPy 3.5
+
+* New features and cleanups
+
+  * 
+  * 
+
+* Bug Fixes 
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+  * 
+
+* RPython improvements
+
+  * 
+  * 
+
+
+.. _here: cpython_differences.html
+
+Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
+======================================================================
+
+* New features
+
+  * 
+  * 
+
+* Bug Fixes
+
+  * 
+  * 
+
+* Performance improvements:
+
+  * 
+  *
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+
+.. _resolved: whatsnew-pypy2-5.9.0.html
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst
rename from pypy/doc/whatsnew-head.rst
rename to pypy/doc/whatsnew-pypy2-5.9.0.rst

From pypy.commits at gmail.com  Sun Sep 24 14:30:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 11:30:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: update version, restart whatsnew-head
Message-ID: <59c7f9c7.c59c1c0a.4ce9c.fbcc@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92456:e2b38e5ae91f
Date: 2017-09-23 23:58 +0300
http://bitbucket.org/pypy/pypy/changeset/e2b38e5ae91f/

Log:	update version, restart whatsnew-head

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-head.rst
@@ -0,0 +1,6 @@
+===========================
+What's new in PyPy2.7 5.10+
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.9.0
+.. startrev:b280d5cca79e 
diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.9.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst
@@ -1,6 +1,6 @@
-==========================
-What's new in PyPy2.7 5.9+
-==========================
+=========================
+What's new in PyPy2.7 5.9
+=========================
 
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8

From pypy.commits at gmail.com  Sun Sep 24 14:30:33 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 11:30:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: resetart whatsnew-pypy3-head (no
 documented changes since 5.8)
Message-ID: <59c7f9c9.0b99df0a.ce4c5.4939@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92457:d9a16970df3c
Date: 2017-09-24 21:12 +0300
http://bitbucket.org/pypy/pypy/changeset/d9a16970df3c/

Log:	resetart whatsnew-pypy3-head (no documented changes since 5.8)

diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.9.0.rst
    whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
@@ -36,6 +37,7 @@
 .. toctree::
 
    whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
 
diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst
@@ -0,0 +1,7 @@
+=======================
+What's new in PyPy3 5.9
+=======================
+
+.. this is the revision after release-pypy3.5-5.8
+.. startrev: afbf09453369
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
 =========================
-What's new in PyPy3 5.7+
+What's new in PyPy3 5.9+
 =========================
 
-.. this is the revision after release-pypy3.3-5.7.x was branched
-.. startrev: afbf09453369
+.. this is the revision after release-pypy3.5-5.9
+.. startrev: be41e3ac0a29
 

From pypy.commits at gmail.com  Sun Sep 24 14:52:56 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 24 Sep 2017 11:52:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix
Message-ID: <59c7ff08.56811c0a.69824.ffb5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92458:8d9b5b766682
Date: 2017-09-24 19:52 +0100
http://bitbucket.org/pypy/pypy/changeset/8d9b5b766682/

Log:	fix

diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
--- a/pypy/module/cpyext/genobject.py
+++ b/pypy/module/cpyext/genobject.py
@@ -1,7 +1,8 @@
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.generator import GeneratorIterator, Coroutine
 from pypy.module.cpyext.api import (
-    build_type_checkers, cts, parse_dir, bootstrap_function)
-from pypy.module.cpyext.pyobject import make_typedescr, as_pyobj
+    build_type_checkers, cts, parse_dir, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import PyObject, make_typedescr, as_pyobj
 from pypy.module.cpyext.object import _dealloc
 
 cts.parse_header(parse_dir / 'cpyext_genobject.h')
@@ -11,7 +12,7 @@
     make_typedescr(GeneratorIterator.typedef,
                    basestruct=cts.gettype('PyGenObject'),
                    attach=gi_attach,
-                   dealloc=_dealloc)
+                   dealloc=gi_dealloc)
 
 
 PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
@@ -19,11 +20,12 @@
 _, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
 
 def gi_attach(space, py_obj, w_obj, w_userdata=None):
-    py_obj.c_gi_code = as_pyobj(space, w_obj.pycode)
+    cts.cast('PyGenObject*', py_obj).c_gi_code = as_pyobj(space, w_obj.pycode)
 
 def gi_realize(space, py_obj):
     raise NotImplementedError(
         "PyPy doesn't support creation of generators from the C-API.")
 
+ at slot_function([PyObject], lltype.Void)
 def gi_dealloc(space, py_obj):
     _dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -127,6 +127,7 @@
 #include "pycapsule.h"
 #include "bytesobject.h"
 #include "sliceobject.h"
+#include "genobject.h"
 #include "datetime.h"
 #include "pystate.h"
 #include "fileobject.h"
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
--- a/pypy/module/cpyext/test/test_genobject.py
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -40,15 +40,16 @@
              else
                 Py_RETURN_FALSE;
              ''')])
+
         def it():
             yield 42
 
         print(module.is_coroutine(it()))
         assert module.is_coroutine(it()) is False
+        self.debug_collect()  # don't crash while deallocating
         from types import coroutine
         assert module.is_coroutine(coroutine(it)()) is True
 
-
     def test_await(self):
         """
         module = self.import_extension('test_coroutine', [

From pypy.commits at gmail.com  Sun Sep 24 16:31:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 13:31:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: update release notes
Message-ID: <59c81608.480f1c0a.b988.cbf8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92459:fb8f57bc42be
Date: 2017-09-24 23:19 +0300
http://bitbucket.org/pypy/pypy/changeset/fb8f57bc42be/

Log:	update release notes

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -81,50 +81,111 @@
 
 * New features and cleanups
 
-  * 
-  * 
+  * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``,
+    ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``,
+    ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat``
+  * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible
+  * ``gcdump`` now reports largest object
+  * More complete support in the ``_curses`` CFFI module
+  * Add cPickle.Unpickler.find_global (issue #1853_)
+  * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set
+  * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()``
+  * Update RevDB to version pypy2.7-v5.6.2
+  * Previously, ``instance.method`` would return always the same bound method
+    object, when gotten from the same instance (as far as ``is`` and ``id()``
+    can tell).  CPython doesn't do that.  Now PyPy, like CPython, returns a 
+    different bound method object every time.  For ``type.method``, PyPy2 still
+    returns always the same *unbound* method object; CPython does it for built-in
+    types but not for user-defined types
+  * Link to disable PaX protection for the JIT when needed
+  * Update build instructions and an rarely used Makefile
+  * Recreate support for using leakfinder in cpyext tests which had suffered
+    bit-rot, disable due to many false positives
+  * Add more functionality to ``sysconfig``
+  * Added ``_swappedbytes_`` support for ``ctypes.Structure``
 
 * Bug Fixes 
 
-  * 
-  * 
+  * Fix issue #2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
+  * Implement ``PyListOjbect.getstorage_copy``
+  * Fix for ``reversed(dictproxy)`` issue #2601_
+  * Fix for duplicate names in ctypes' ``_fields__``, issue #2621_
+  * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
+  * ``gc.get_objects`` now handles objects with finalizers more consistently
+
+* Performance improvements:
+
+  * Improve performance of ``bytearray.extend`` by rewriting portions in app-level
+  * Optimize list accesses with constant indexes better by retaining more
+    information about them
+  * Add a jit driver for ``array.count`` and ``array.index``
+  * Improve information retained in a bridge wrt ``array``
+  * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into
+    pure C macros
+  * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
+    all the ints
+
+* RPython improvements
+
+  * Do not preallocate a RPython list if we only know an upper bound on its size
+  * Issue #2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
+  * Replace magical NOT RPYTHON comment with a decorator
+  * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
+
+* Degredations
+
+  * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module
+
+.. _here: cpython_differences.html
+.. _1853: https://bitbucket.org/pypy/pypy/issues/1853
+.. _2592: https://bitbucket.org/pypy/pypy/issues/2592
+.. _2590: https://bitbucket.org/pypy/pypy/issues/2590
+.. _2621: https://bitbucket.org/pypy/pypy/issues/2621
+
+Highlights of the PyPy3.5 release (since 5.8 beta released June 2017)
+======================================================================
+
+* New features
+
+  * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
+    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``
+  * mplement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
+  * Support the new version of ``os.stat()`` on win32
+  * Use ``stat3()`` on Posix
+  * Accept buffer objects as filenames, except for `oslistdir``
+  * Make slices of array ``memoryview`` s usable as writable buffers if contiguous
+  * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded
+  * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version
+  * nsure that ``mappingproxy`` is recognised as a mapping, not a sequence
+  * Enable PGO for CLang
+  * Rework ``cppyy`` packaging and rename the backend to ``_cppyy``
+  * Support for libressl 2.5.4
+  * Mirror CPython ``classmethod __reduce__`` which fixes pickling test
+  * Use utf-8 for ``readline`` history file
+  * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses
+
+* Bug Fixes
+
+  * Try to make ``openssl`` CFFI bindings more general and future-proof
+  * Better support ``importlib`` by only listing built-in modules in ``sys.builtin``
+  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue #2579_
+  * Fix for ``reversed(mapping object)`` issue #2601_
+  * Fixing regression with non-started generator receiving non-``None``, should
+    always raise ``TypeError``
+  * ``itertools.islice``: use same logic as CPython, fixes #2643_
 
 * Performance improvements:
 
   * 
-  * 
-
-* RPython improvements
-
-  * 
-  * 
-
-
-.. _here: cpython_differences.html
-
-Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
-======================================================================
-
-* New features
-
-  * 
-  * 
-
-* Bug Fixes
-
-  * 
-  * 
-
-* Performance improvements:
-
-  * 
-  *
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
 
 .. _resolved: whatsnew-pypy2-5.9.0.html
+.. _2579: https://bitbucket.org/pypy/pypy/issues/2579
+.. _2601: https://bitbucket.org/pypy/pypy/issues/2601
+.. _2643: https://bitbucket.org/pypy/pypy/issues/2643
 
 Please update, and continue to help us make PyPy better.
 

From pypy.commits at gmail.com  Sun Sep 24 16:31:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 13:31:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: import earlier
Message-ID: <59c8160a.c1a31c0a.a4a1d.a149@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92460:4fe50ed6e2ac
Date: 2017-09-24 23:19 +0300
http://bitbucket.org/pypy/pypy/changeset/4fe50ed6e2ac/

Log:	import earlier

diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -214,7 +214,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)

From pypy.commits at gmail.com  Sun Sep 24 16:31:08 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 13:31:08 -0700 (PDT)
Subject: [pypy-commit] pypy win32-fixes6: close abandoned branch
Message-ID: <59c8160c.51421c0a.f4f40.2c48@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-fixes6
Changeset: r92461:b8233cc7d857
Date: 2017-09-24 23:20 +0300
http://bitbucket.org/pypy/pypy/changeset/b8233cc7d857/

Log:	close abandoned branch


From pypy.commits at gmail.com  Sun Sep 24 16:31:10 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 13:31:10 -0700 (PDT)
Subject: [pypy-commit] pypy default: finish reviewing commits up to this one
Message-ID: <59c8160e.75afdf0a.ea78.495b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92462:646e49bc2489
Date: 2017-09-24 23:30 +0300
http://bitbucket.org/pypy/pypy/changeset/646e49bc2489/

Log:	finish reviewing commits up to this one

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -103,6 +103,7 @@
     bit-rot, disable due to many false positives
   * Add more functionality to ``sysconfig``
   * Added ``_swappedbytes_`` support for ``ctypes.Structure``
+  * Better support the ``inspect`` module on ``frames``
 
 * Bug Fixes 
 
@@ -124,6 +125,7 @@
     pure C macros
   * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
     all the ints
+  * Cache string keys that occur in json dicts, as they are likely to repeat
 
 * RPython improvements
 
@@ -148,7 +150,8 @@
 * New features
 
   * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
-    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``
+    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
+    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``
   * mplement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
   * Support the new version of ``os.stat()`` on win32
   * Use ``stat3()`` on Posix
@@ -163,6 +166,7 @@
   * Mirror CPython ``classmethod __reduce__`` which fixes pickling test
   * Use utf-8 for ``readline`` history file
   * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses
+  * Add async slot functions in cpyext
 
 * Bug Fixes
 

From pypy.commits at gmail.com  Sun Sep 24 22:01:48 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 24 Sep 2017 19:01:48 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix translation
Message-ID: <59c8638c.cd5e1c0a.bac78.72c5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92463:ca8b2b8a0df0
Date: 2017-09-25 03:01 +0100
http://bitbucket.org/pypy/pypy/changeset/ca8b2b8a0df0/

Log:	fix translation

diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
--- a/pypy/module/cpyext/genobject.py
+++ b/pypy/module/cpyext/genobject.py
@@ -20,6 +20,7 @@
 _, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
 
 def gi_attach(space, py_obj, w_obj, w_userdata=None):
+    assert isinstance(w_obj, GeneratorIterator)
     cts.cast('PyGenObject*', py_obj).c_gi_code = as_pyobj(space, w_obj.pycode)
 
 def gi_realize(space, py_obj):

From pypy.commits at gmail.com  Mon Sep 25 01:33:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 22:33:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix tag,
 fix PYPY_VERSION_NUM is hex not decimal
Message-ID: <59c89530.c55c1c0a.271e.cc81@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92464:1fb2a6a3b647
Date: 2017-09-25 08:26 +0300
http://bitbucket.org/pypy/pypy/changeset/1fb2a6a3b647/

Log:	fix tag, fix PYPY_VERSION_NUM is hex not decimal

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,4 +3,4 @@
 ===========================
 
 .. this is a revision shortly after release-pypy2.7-v5.9.0
-.. startrev:b280d5cca79e 
+.. startrev:899e5245de1e
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -30,7 +30,7 @@
 
 /* PyPy version as a string */
 #define PYPY_VERSION "5.10.0-alpha0"
-#define PYPY_VERSION_NUM  0x05100000
+#define PYPY_VERSION_NUM  0x050A0000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object

From pypy.commits at gmail.com  Mon Sep 25 01:33:38 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 22:33:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: CMSG_SPACE,
 CMSG_LEN not available on windows
Message-ID: <59c89532.03bcdf0a.30960.140e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92465:6a7983eb68b8
Date: 2017-09-25 08:32 +0300
http://bitbucket.org/pypy/pypy/changeset/6a7983eb68b8/

Log:	CMSG_SPACE, CMSG_LEN not available on windows

diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
 from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
@@ -327,41 +328,42 @@
             for (family, socktype, protocol, canonname, addr) in lst]
     return space.newlist(lst1)
 
- at unwrap_spec(size=int)
-def CMSG_SPACE(space, size):
-    """
-    Socket method to determine the optimal byte size of the ancillary.
-    Recommended to be used when computing the ancillary size for recvmsg.
-    :param space:
-    :param size: an integer with the minimum size required.
-    :return: an integer with the minimum memory needed for the required size. The value is memory alligned
-    """
-    if size < 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_SPACE() argument out of range")
-    retval = rsocket.CMSG_SPACE(size)
-    if retval == 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_SPACE() argument out of range")
-    return space.newint(retval)
+if sys.platform != 'win32':
+    @unwrap_spec(size=int)
+    def CMSG_SPACE(space, size):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param space:
+        :param size: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is memory alligned
+        """
+        if size < 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_SPACE() argument out of range")
+        retval = rsocket.CMSG_SPACE(size)
+        if retval == 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_SPACE() argument out of range")
+        return space.newint(retval)
 
- at unwrap_spec(len=int)
-def CMSG_LEN(space, len):
-    """
-    Socket method to determine the optimal byte size of the ancillary.
-    Recommended to be used when computing the ancillary size for recvmsg.
-    :param space:
-    :param len: an integer with the minimum size required.
-    :return: an integer with the minimum memory needed for the required size. The value is not mem alligned.
-    """
-    if len < 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_LEN() argument out of range")
-    retval = rsocket.CMSG_LEN(len)
-    if retval == 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_LEN() argument out of range")
-    return space.newint(retval)
+    @unwrap_spec(len=int)
+    def CMSG_LEN(space, len):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param space:
+        :param len: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is not mem alligned.
+        """
+        if len < 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_LEN() argument out of range")
+        retval = rsocket.CMSG_LEN(len)
+        if retval == 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_LEN() argument out of range")
+        return space.newint(retval)
 
 def getdefaulttimeout(space):
     """getdefaulttimeout() -> timeout

From pypy.commits at gmail.com  Mon Sep 25 02:45:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Sun, 24 Sep 2017 23:45:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: update release notes
Message-ID: <59c8a5f2.c1acdf0a.36c1.a009@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92466:61e579f98356
Date: 2017-09-25 09:44 +0300
http://bitbucket.org/pypy/pypy/changeset/61e579f98356/

Log:	update release notes

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -10,19 +10,27 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-XXXX hilights here
+Only a handful of failing tests remain in NumPy and Pandas on PyPy2.7, issues
+that appeared as excessive memory use were cleared up and other incompatibilities
+were resolved.
 
+Cython 2.7 (released last week) should support more projects with PyPy, both on
+PyPy2.7 and PyPy3.5 beta.
+
+We improved JSON support for recurring string keys when parsing, which should
+decrease memory use and increase parsing speed.
+
+CFFI_, which is part of the PyPy release, has been updated to 1.11,
+improving an already great package for interfacing with C. CFFI now supports
+complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has
+improved support for callbacks.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
-CPython were ported to PyPy and PEP 489 was fully implemented. Of course the
-bug fixes and performance enhancements mentioned above are part of both PyPy
-2.7 and PyPy 3.5.
-
-CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.11,
-improving an already great package for interfacing with C.
+CPython were ported to PyPy. Of course the bug fixes and performance enhancements
+mentioned above are part of both PyPy2.7 and PyPy3.5 beta.
 
 As always, this release fixed many other issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -125,7 +133,7 @@
     pure C macros
   * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
     all the ints
-  * Cache string keys that occur in json dicts, as they are likely to repeat
+  * Cache string keys that occur in JSON dicts, as they are likely to repeat
 
 * RPython improvements
 

From pypy.commits at gmail.com  Mon Sep 25 09:03:22 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 25 Sep 2017 06:03:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: some numbers
Message-ID: <59c8fe9a.6588df0a.96a68.54fa@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: 
Changeset: r92467:dc8707606e3e
Date: 2017-09-25 15:02 +0200
http://bitbucket.org/pypy/pypy/changeset/dc8707606e3e/

Log:	some numbers

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -17,8 +17,9 @@
 Cython 2.7 (released last week) should support more projects with PyPy, both on
 PyPy2.7 and PyPy3.5 beta.
 
-We improved JSON support for recurring string keys when parsing, which should
-decrease memory use and increase parsing speed.
+We optimized the JSON parser for recurring string keys, which should decrease
+memory use to 50% and increase parsing speed by up to 15% for large JSON files
+with many repeating dictionary keys (which is quite common).
 
 CFFI_, which is part of the PyPy release, has been updated to 1.11,
 improving an already great package for interfacing with C. CFFI now supports

From pypy.commits at gmail.com  Mon Sep 25 13:52:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 25 Sep 2017 10:52:36 -0700 (PDT)
Subject: [pypy-commit] pypy win32-slow-tests: instrument and compare linux
 to win32 on the same machine (not under a vm)
Message-ID: <59c94264.a7a6df0a.e4867.8ed2@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: win32-slow-tests
Changeset: r92468:d8fd9d7c8d1b
Date: 2017-09-25 20:42 +0300
http://bitbucket.org/pypy/pypy/changeset/d8fd9d7c8d1b/

Log:	instrument and compare linux to win32 on the same machine (not under
	a vm)

diff --git a/rpython/translator/c/test/test_typed.py b/rpython/translator/c/test/test_typed.py
--- a/rpython/translator/c/test/test_typed.py
+++ b/rpython/translator/c/test/test_typed.py
@@ -1,7 +1,7 @@
 from __future__ import with_statement
 
 import math
-import sys, os
+import sys, os, time
 import py
 
 from rpython.rlib.rstackovf import StackOverflow
@@ -468,14 +468,27 @@
         fn(0, expected_exception_name='ZeroDivisionError')
 
     def test_int_mul_ovf(self):
+        tic = time.clock()
         fn = self.getcompiled(snippet.mul_func, [int, int])
+        toc = time.clock()
+        compiletime = toc - tic
+        tic = toc
         for y in range(-5, 5):
             for x in range(-5, 5):
                 assert fn(x, y) == snippet.mul_func(x, y)
         n = sys.maxint / 4
         assert fn(n, 3) == snippet.mul_func(n, 3)
         assert fn(n, 4) == snippet.mul_func(n, 4)
+        toc = time.clock()
+        runningtime = toc - tic
+        tic = toc
         fn(n, 5, expected_exception_name='OverflowError')
+        toc = time.clock()
+        exceptiontime = toc - tic
+        tic = toc
+        print 'compiletime,runningtime,exceptiontime',compiletime,runningtime,exceptiontime
+        # same machine - linux 1.168619     0.026848      0.000303
+        #                win32 8.2197742749 3.80634991368 0.0371649693735
 
     def test_int_mod_ovf_zer(self):
         fn = self.getcompiled(snippet.mod_func, [int])

From pypy.commits at gmail.com  Mon Sep 25 13:52:38 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 25 Sep 2017 10:52:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: typos and links (thanks to reviewers on
 pypy-dev)
Message-ID: <59c94266.4e301c0a.70d33.7967@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92469:3a589f5a003b
Date: 2017-09-25 20:49 +0300
http://bitbucket.org/pypy/pypy/changeset/3a589f5a003b/

Log:	typos and links (thanks to reviewers on pypy-dev)

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -14,8 +14,8 @@
 that appeared as excessive memory use were cleared up and other incompatibilities
 were resolved.
 
-Cython 2.7 (released last week) should support more projects with PyPy, both on
-PyPy2.7 and PyPy3.5 beta.
+Cython 0.27 (released last week) should support more projects with PyPy, both
+on PyPy2.7 and PyPy3.5 beta.
 
 We optimized the JSON parser for recurring string keys, which should decrease
 memory use to 50% and increase parsing speed by up to 15% for large JSON files
@@ -96,7 +96,7 @@
   * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible
   * ``gcdump`` now reports largest object
   * More complete support in the ``_curses`` CFFI module
-  * Add cPickle.Unpickler.find_global (issue #1853_)
+  * Add cPickle.Unpickler.find_global (issue 1853_)
   * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set
   * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()``
   * Update RevDB to version pypy2.7-v5.6.2
@@ -116,10 +116,10 @@
 
 * Bug Fixes 
 
-  * Fix issue #2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
+  * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
   * Implement ``PyListOjbect.getstorage_copy``
-  * Fix for ``reversed(dictproxy)`` issue #2601_
-  * Fix for duplicate names in ctypes' ``_fields__``, issue #2621_
+  * Fix for ``reversed(dictproxy)`` issue 2601_
+  * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_
   * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
   * ``gc.get_objects`` now handles objects with finalizers more consistently
 
@@ -139,7 +139,7 @@
 * RPython improvements
 
   * Do not preallocate a RPython list if we only know an upper bound on its size
-  * Issue #2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
+  * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
   * Replace magical NOT RPYTHON comment with a decorator
   * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
 
@@ -161,14 +161,14 @@
   * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
     ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
     ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``
-  * mplement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
+  * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
   * Support the new version of ``os.stat()`` on win32
   * Use ``stat3()`` on Posix
   * Accept buffer objects as filenames, except for `oslistdir``
   * Make slices of array ``memoryview`` s usable as writable buffers if contiguous
   * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded
   * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version
-  * nsure that ``mappingproxy`` is recognised as a mapping, not a sequence
+  * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence
   * Enable PGO for CLang
   * Rework ``cppyy`` packaging and rename the backend to ``_cppyy``
   * Support for libressl 2.5.4
@@ -181,11 +181,11 @@
 
   * Try to make ``openssl`` CFFI bindings more general and future-proof
   * Better support ``importlib`` by only listing built-in modules in ``sys.builtin``
-  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue #2579_
-  * Fix for ``reversed(mapping object)`` issue #2601_
+  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_
+  * Fix for ``reversed(mapping object)`` issue 2601_
   * Fixing regression with non-started generator receiving non-``None``, should
     always raise ``TypeError``
-  * ``itertools.islice``: use same logic as CPython, fixes #2643_
+  * ``itertools.islice``: use same logic as CPython, fixes 2643_
 
 * Performance improvements:
 

From pypy.commits at gmail.com  Mon Sep 25 17:20:47 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Mon, 25 Sep 2017 14:20:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: blurb a18e6d5
Message-ID: <59c9732f.1986df0a.5bca8.a658@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r92470:4bda0b81a960
Date: 2017-09-25 14:20 -0700
http://bitbucket.org/pypy/pypy/changeset/4bda0b81a960/

Log:	blurb a18e6d5

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -122,6 +122,9 @@
   * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_
   * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
   * ``gc.get_objects`` now handles objects with finalizers more consistently
+  * Fixed memory leak in ``SSLContext.getpeercert`` returning validated
+    certificates and ``SSLContext.get_ca_certs(binary_mode=True)``
+    (_get_crl_dp) `CPython issue 29738`_
 
 * Performance improvements:
 
@@ -199,6 +202,7 @@
 .. _2579: https://bitbucket.org/pypy/pypy/issues/2579
 .. _2601: https://bitbucket.org/pypy/pypy/issues/2601
 .. _2643: https://bitbucket.org/pypy/pypy/issues/2643
+.. _CPython issue 29738: https://bugs.python.org/issue29738
 
 Please update, and continue to help us make PyPy better.
 

From pypy.commits at gmail.com  Tue Sep 26 02:41:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 25 Sep 2017 23:41:09 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: PyUnicode_Substring()
Message-ID: <59c9f685.84891c0a.2a5a9.a0d0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92471:8b9aafe3487b
Date: 2017-09-26 08:40 +0200
http://bitbucket.org/pypy/pypy/changeset/8b9aafe3487b/

Log:	PyUnicode_Substring()

diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -851,6 +851,17 @@
         assert r"['a\n', 'b\n', 'c\n', 'd']" == space.unwrap(space.repr(
                 PyUnicode_Splitlines(space, w_str, 1)))
 
+    def test_substring_api(self, space):
+        w_str = space.wrap(u"abcd")
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 1, 3)) == u"bc"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 0, 4)) == u"abcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 0, 9)) == u"abcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 1, 4)) == u"bcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 2, 2)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 5, 4)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 5, 3)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 4, 3)) == u""
+
     def test_Ready(self, space):
         w_str = space.wrap(u'abc')  # ASCII
         py_str = as_pyobj(space, w_str)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1043,3 +1043,17 @@
     resulting strings."""
     w_keepend = space.newbool(bool(rffi.cast(lltype.Signed, keepend)))
     return space.call_method(w_str, "splitlines", w_keepend)
+
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyUnicode_Substring(space, w_str, start, end):
+    usrc = space.unicode_w(w_str)
+    length = len(usrc)
+    if start < 0 or end < 0:
+        raise oefmt(space.w_IndexError, "string index out of range")
+    if start >= length or end < start:
+        result = u''
+    else:
+        if end > length:
+            end = length
+        result = usrc[start:end]
+    return space.newunicode(result)

From pypy.commits at gmail.com  Tue Sep 26 02:47:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 25 Sep 2017 23:47:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: PyLong_FromUnicodeObject()
Message-ID: <59c9f7fb.d5121c0a.fec95.b585@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r92472:d350c77746fc
Date: 2017-09-26 08:46 +0200
http://bitbucket.org/pypy/pypy/changeset/d350c77746fc/

Log:	PyLong_FromUnicodeObject()

diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -201,6 +201,10 @@
     for the conversion.  The radix must be in the range [2, 36]; if it is
     out of range, ValueError will be raised."""
     w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
+    return PyLong_FromUnicodeObject(space, w_value, base)
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyLong_FromUnicodeObject(space, w_value, base):
     w_base = space.newint(rffi.cast(lltype.Signed, base))
     return space.call_function(space.w_long, w_value, w_base)
 
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -272,6 +272,19 @@
         # A string with arabic digits. 'BAD' is after the 6th character.
         assert module.from_unicode(u'  1\u0662\u0663\u0664BAD') == (1234, 4660)
 
+    def test_fromunicodeobject(self):
+        module = self.import_extension('foo', [
+            ("from_unicodeobject", "METH_O",
+             """
+                 return Py_BuildValue("NN",
+                     PyLong_FromUnicodeObject(args, 10),
+                     PyLong_FromUnicodeObject(args, 16));
+             """),
+            ])
+        # A string with arabic digits.
+        assert (module.from_unicodeobject(u'  1\u0662\u0663\u0664')
+                == (1234, 4660))
+
     def test_aslong(self):
         module = self.import_extension('foo', [
             ("as_long", "METH_O",

From pypy.commits at gmail.com  Wed Sep 27 02:24:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 26 Sep 2017 23:24:15 -0700 (PDT)
Subject: [pypy-commit] cffi default: Apply the same patch as everyone else
Message-ID: <59cb440f.c7341c0a.b9ce9.c345@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3024:4b0741f26fcd
Date: 2017-09-27 08:23 +0200
http://bitbucket.org/cffi/cffi/changeset/4b0741f26fcd/

Log:	Apply the same patch as everyone else

diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -237,6 +237,14 @@
 
 #if PY_MAJOR_VERSION >= 3
 /* pfffffffffffff pages of copy-paste from listobject.c */
+
+/* pfffffffffffff#2: the PySlice_GetIndicesEx() *macro* should not
+   be called, because C extension modules compiled with it differ
+   on ABI between 3.6.0, 3.6.1 and 3.6.2. */
+#if PY_VERSION_HEX < 0x03070000 && defined(PySlice_GetIndicesEx) && !defined(PYPY_VERSION)
+#undef PySlice_GetIndicesEx
+#endif
+
 static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item)
 {
     if (PyIndex_Check(item)) {

From pypy.commits at gmail.com  Wed Sep 27 03:29:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 00:29:08 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document 4b0741f26fcd
Message-ID: <59cb5344.579c1c0a.4fa11.1281@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3025:23fa387f3e05
Date: 2017-09-27 09:23 +0200
http://bitbucket.org/cffi/cffi/changeset/23fa387f3e05/

Log:	Document 4b0741f26fcd

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -8,6 +8,11 @@
 
 * Fix tests, remove deprecated C API usage
 
+* Fix for 3.6.0/3.6.1 being incompatible with each other unless we hack
+  (cpython issue `#29943`_)
+  
+.. _`#29943`: https://bugs.python.org/issue29943
+
 
 v1.11
 =====

From pypy.commits at gmail.com  Wed Sep 27 03:29:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 00:29:10 -0700 (PDT)
Subject: [pypy-commit] cffi default: Update version to 1.11.1
Message-ID: <59cb5346.41621c0a.3fedc.52e6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3026:32ecdbdba707
Date: 2017-09-27 09:28 +0200
http://bitbucket.org/cffi/cffi/changeset/32ecdbdba707/

Log:	Update version to 1.11.1

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.11.0"
+#define CFFI_VERSION  "1.11.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.11'
 # The full version, including alpha/beta/rc tags.
-release = '1.11.0'
+release = '1.11.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* https://pypi.python.org/packages/4e/32/4070bdf32812c89eb635c80880a5caa2e0189aa7999994c265577e5154f3/cffi-1.11.0.tar.gz#md5=2c5939cc2fa0183fe0c2fcb6a4f1994d
+* https://pypi.python.org/pypi/cffi
 
-   - MD5: 2c5939cc2fa0183fe0c2fcb6a4f1994d
+   - MD5: ...
 
-   - SHA: 93cb5aaf152e19f9d4082a723aa2396e9cd5d93f
+   - SHA: ...
 
-   - SHA256: 5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -186,7 +186,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.11.0',
+        version='1.11.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}

From pypy.commits at gmail.com  Wed Sep 27 03:37:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 00:37:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Update to cffi 1.11.1
Message-ID: <59cb5546.ba87df0a.8bff4.bf80@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92473:a3a897f9b96e
Date: 2017-09-27 09:36 +0200
http://bitbucket.org/pypy/pypy/changeset/a3a897f9b96e/

Log:	Update to cffi 1.11.1

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.11.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Wed Sep 27 03:38:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 00:38:31 -0700 (PDT)
Subject: [pypy-commit] cffi default: "char *" => "const char *" until Python
 3.7 is happy
Message-ID: <59cb5577.05d21c0a.e798.faf2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3027:a5f702205e81
Date: 2017-09-27 09:38 +0200
http://bitbucket.org/cffi/cffi/changeset/a5f702205e81/

Log:	"char *" => "const char *" until Python 3.7 is happy

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2675,7 +2675,7 @@
 static void
 _cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr)
 {
-    char *text;
+    const char *text;
     if (!PyErr_ExceptionMatches(PyExc_AttributeError))
         return;
     PyErr_Clear();
@@ -5900,7 +5900,7 @@
         if (!PyText_Check(tmpkey)) {
 #if PY_MAJOR_VERSION < 3
             if (PyUnicode_Check(tmpkey)) {
-                char *text = PyText_AsUTF8(tmpkey);
+                const char *text = PyText_AsUTF8(tmpkey);
                 if (text == NULL)
                     goto error;
                 Py_DECREF(tmpkey);
diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -51,7 +51,7 @@
 
 static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn)
 {
-    char *s;
+    const char *s;
     PyObject *error, *onerror, *infotuple, *old1;
     int index, err;
     const struct _cffi_global_s *g;
diff --git a/c/cdlopen.c b/c/cdlopen.c
--- a/c/cdlopen.c
+++ b/c/cdlopen.c
@@ -1,6 +1,7 @@
 /* ffi.dlopen() interface with dlopen()/dlsym()/dlclose() */
 
-static void *cdlopen_fetch(PyObject *libname, void *libhandle, char *symbol)
+static void *cdlopen_fetch(PyObject *libname, void *libhandle,
+                           const char *symbol)
 {
     void *address;
 
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -92,7 +92,7 @@
 /* forward, declared in cdlopen.c because it's mostly useful for this case */
 static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds);
 
-static PyObject *ffi_fetch_int_constant(FFIObject *ffi, char *name,
+static PyObject *ffi_fetch_int_constant(FFIObject *ffi, const char *name,
                                         int recursion)
 {
     int index;
@@ -145,7 +145,7 @@
 #define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
 #define CONSIDER_FN_AS_FNPTR  8
 
-static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, char *input_text)
+static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, const char *input_text)
 {
     size_t length = strlen(input_text);
     char *extra;
@@ -188,7 +188,7 @@
         PyObject *x = PyDict_GetItem(types_dict, arg);
 
         if (x == NULL) {
-            char *input_text = PyText_AS_UTF8(arg);
+            const char *input_text = PyText_AS_UTF8(arg);
             int err, index = parse_c_type(&ffi->info, input_text);
             if (index < 0)
                 return _ffi_bad_type(ffi, input_text);
diff --git a/c/file_emulator.h b/c/file_emulator.h
--- a/c/file_emulator.h
+++ b/c/file_emulator.h
@@ -33,7 +33,7 @@
     PyObject *ob, *ob_capsule = NULL, *ob_mode = NULL;
     FILE *f;
     int fd;
-    char *mode;
+    const char *mode;
 
     ob = PyObject_CallMethod(ob_file, "flush", NULL);
     if (ob == NULL)
diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -85,7 +85,8 @@
 }
 
 static void cdlopen_close_ignore_errors(void *libhandle);  /* forward */
-static void *cdlopen_fetch(PyObject *libname, void *libhandle, char *symbol);
+static void *cdlopen_fetch(PyObject *libname, void *libhandle,
+                           const char *symbol);
 
 static void lib_dealloc(LibObject *lib)
 {
@@ -127,7 +128,7 @@
     int i, type_index = _CFFI_GETARG(g->type_op);
     _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types;
     static const char *const format = ";\n\nCFFI C function from %s.lib";
-    char *libname = PyText_AS_UTF8(lib->l_libname);
+    const char *libname = PyText_AS_UTF8(lib->l_libname);
     struct funcbuilder_s funcbuilder;
 
     /* return type: */
@@ -206,7 +207,7 @@
     const struct _cffi_global_s *g;
     CTypeDescrObject *ct;
     builder_c_t *types_builder = lib->l_types_builder;
-    char *s = PyText_AsUTF8(name);
+    const char *s = PyText_AsUTF8(name);
     if (s == NULL)
         return NULL;
 
@@ -493,7 +494,7 @@
 
 static PyObject *lib_getattr(LibObject *lib, PyObject *name)
 {
-    char *p;
+    const char *p;
     PyObject *x;
     LIB_GET_OR_CACHE_ADDR(x, lib, name, goto missing);
 

From pypy.commits at gmail.com  Wed Sep 27 06:38:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 03:38:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: Attempt to hack for
 https://bugs.python.org/issue29943
Message-ID: <59cb7f9f.9c471c0a.a0b22.8bf4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92474:1495dbdd3cfd
Date: 2017-09-27 12:37 +0200
http://bitbucket.org/pypy/pypy/changeset/1495dbdd3cfd/

Log:	Attempt to hack for https://bugs.python.org/issue29943

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1314,6 +1314,18 @@
     decls = defaultdict(list)
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
+    decls[pypy_decl].append("""
+        /* hack for https://bugs.python.org/issue29943 */
+        PyAPI_FUNC(int) %s(PySliceObject *arg0,
+                           Signed arg1, Signed *arg2,
+                           Signed *arg3, Signed *arg4, Signed *arg5);
+        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+                Py_ssize_t *arg5) {
+            return %s(arg0, arg1, arg2, arg3,
+                      arg4, arg5);
+        }
+    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]

From pypy.commits at gmail.com  Wed Sep 27 06:51:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 03:51:14 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.11: hg merge default
Message-ID: <59cb82a2.e393df0a.cfe1e.0ffc@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.11
Changeset: r3028:8adcf995d67c
Date: 2017-09-27 12:50 +0200
http://bitbucket.org/cffi/cffi/changeset/8adcf995d67c/

Log:	hg merge default

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -2,7 +2,7 @@
 #include <Python.h>
 #include "structmember.h"
 
-#define CFFI_VERSION  "1.11.0"
+#define CFFI_VERSION  "1.11.1"
 
 #ifdef MS_WIN32
 #include <windows.h>
@@ -101,7 +101,11 @@
 # define PyText_FromFormat PyUnicode_FromFormat
 # define PyText_AsUTF8 _PyUnicode_AsString   /* PyUnicode_AsUTF8 in Py3.3 */
 # define PyText_AS_UTF8 _PyUnicode_AsString
-# define PyText_GetSize PyUnicode_GetSize
+# if PY_VERSION_HEX >= 0x03030000
+#  define PyText_GetSize PyUnicode_GetLength
+# else
+#  define PyText_GetSize PyUnicode_GetSize
+# endif
 # define PyText_FromString PyUnicode_FromString
 # define PyText_FromStringAndSize PyUnicode_FromStringAndSize
 # define PyText_InternInPlace PyUnicode_InternInPlace
@@ -2671,7 +2675,7 @@
 static void
 _cdata_attr_errmsg(char *errmsg, CDataObject *cd, PyObject *attr)
 {
-    char *text;
+    const char *text;
     if (!PyErr_ExceptionMatches(PyExc_AttributeError))
         return;
     PyErr_Clear();
@@ -5896,7 +5900,7 @@
         if (!PyText_Check(tmpkey)) {
 #if PY_MAJOR_VERSION < 3
             if (PyUnicode_Check(tmpkey)) {
-                char *text = PyText_AsUTF8(tmpkey);
+                const char *text = PyText_AsUTF8(tmpkey);
                 if (text == NULL)
                     goto error;
                 Py_DECREF(tmpkey);
diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -51,7 +51,7 @@
 
 static PyObject *_ffi_def_extern_decorator(PyObject *outer_args, PyObject *fn)
 {
-    char *s;
+    const char *s;
     PyObject *error, *onerror, *infotuple, *old1;
     int index, err;
     const struct _cffi_global_s *g;
diff --git a/c/cdlopen.c b/c/cdlopen.c
--- a/c/cdlopen.c
+++ b/c/cdlopen.c
@@ -1,6 +1,7 @@
 /* ffi.dlopen() interface with dlopen()/dlsym()/dlclose() */
 
-static void *cdlopen_fetch(PyObject *libname, void *libhandle, char *symbol)
+static void *cdlopen_fetch(PyObject *libname, void *libhandle,
+                           const char *symbol)
 {
     void *address;
 
diff --git a/c/ffi_obj.c b/c/ffi_obj.c
--- a/c/ffi_obj.c
+++ b/c/ffi_obj.c
@@ -92,7 +92,7 @@
 /* forward, declared in cdlopen.c because it's mostly useful for this case */
 static int ffiobj_init(PyObject *self, PyObject *args, PyObject *kwds);
 
-static PyObject *ffi_fetch_int_constant(FFIObject *ffi, char *name,
+static PyObject *ffi_fetch_int_constant(FFIObject *ffi, const char *name,
                                         int recursion)
 {
     int index;
@@ -145,7 +145,7 @@
 #define ACCEPT_ALL      (ACCEPT_STRING | ACCEPT_CTYPE | ACCEPT_CDATA)
 #define CONSIDER_FN_AS_FNPTR  8
 
-static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, char *input_text)
+static CTypeDescrObject *_ffi_bad_type(FFIObject *ffi, const char *input_text)
 {
     size_t length = strlen(input_text);
     char *extra;
@@ -188,7 +188,7 @@
         PyObject *x = PyDict_GetItem(types_dict, arg);
 
         if (x == NULL) {
-            char *input_text = PyText_AS_UTF8(arg);
+            const char *input_text = PyText_AS_UTF8(arg);
             int err, index = parse_c_type(&ffi->info, input_text);
             if (index < 0)
                 return _ffi_bad_type(ffi, input_text);
diff --git a/c/file_emulator.h b/c/file_emulator.h
--- a/c/file_emulator.h
+++ b/c/file_emulator.h
@@ -33,7 +33,7 @@
     PyObject *ob, *ob_capsule = NULL, *ob_mode = NULL;
     FILE *f;
     int fd;
-    char *mode;
+    const char *mode;
 
     ob = PyObject_CallMethod(ob_file, "flush", NULL);
     if (ob == NULL)
diff --git a/c/lib_obj.c b/c/lib_obj.c
--- a/c/lib_obj.c
+++ b/c/lib_obj.c
@@ -85,7 +85,8 @@
 }
 
 static void cdlopen_close_ignore_errors(void *libhandle);  /* forward */
-static void *cdlopen_fetch(PyObject *libname, void *libhandle, char *symbol);
+static void *cdlopen_fetch(PyObject *libname, void *libhandle,
+                           const char *symbol);
 
 static void lib_dealloc(LibObject *lib)
 {
@@ -127,7 +128,7 @@
     int i, type_index = _CFFI_GETARG(g->type_op);
     _cffi_opcode_t *opcodes = lib->l_types_builder->ctx.types;
     static const char *const format = ";\n\nCFFI C function from %s.lib";
-    char *libname = PyText_AS_UTF8(lib->l_libname);
+    const char *libname = PyText_AS_UTF8(lib->l_libname);
     struct funcbuilder_s funcbuilder;
 
     /* return type: */
@@ -206,7 +207,7 @@
     const struct _cffi_global_s *g;
     CTypeDescrObject *ct;
     builder_c_t *types_builder = lib->l_types_builder;
-    char *s = PyText_AsUTF8(name);
+    const char *s = PyText_AsUTF8(name);
     if (s == NULL)
         return NULL;
 
@@ -493,7 +494,7 @@
 
 static PyObject *lib_getattr(LibObject *lib, PyObject *name)
 {
-    char *p;
+    const char *p;
     PyObject *x;
     LIB_GET_OR_CACHE_ADDR(x, lib, name, goto missing);
 
@@ -530,6 +531,13 @@
         PyErr_Clear();
         return PyText_FromFormat("%s.lib", PyText_AS_UTF8(lib->l_libname));
     }
+#if PY_MAJOR_VERSION >= 3
+    if (strcmp(p, "__loader__") == 0 || strcmp(p, "__spec__") == 0) {
+        /* some more module-like behavior hacks */
+        Py_INCREF(Py_None);
+        return Py_None;
+    }
+#endif
     return NULL;
 }
 
diff --git a/c/minibuffer.h b/c/minibuffer.h
--- a/c/minibuffer.h
+++ b/c/minibuffer.h
@@ -56,14 +56,17 @@
     }
 }
 
+/* forward: from _cffi_backend.c */
+static int _fetch_as_buffer(PyObject *x, Py_buffer *view, int writable_only);
+
 static int mb_ass_slice(MiniBufferObj *self,
                         Py_ssize_t left, Py_ssize_t right, PyObject *other)
 {
-    const void *buffer;
-    Py_ssize_t buffer_len, count;
+    Py_ssize_t count;
     Py_ssize_t size = self->mb_size;
+    Py_buffer src_view;
 
-    if (PyObject_AsReadBuffer(other, &buffer, &buffer_len) < 0)
+    if (_fetch_as_buffer(other, &src_view, 0) < 0)
         return -1;
 
     if (left < 0)     left = 0;
@@ -71,12 +74,14 @@
     if (left > right) left = right;
 
     count = right - left;
-    if (count != buffer_len) {
+    if (count != src_view.len) {
+        PyBuffer_Release(&src_view);
         PyErr_SetString(PyExc_ValueError,
                         "right operand length must match slice length");
         return -1;
     }
-    memcpy(self->mb_data + left, buffer, count);
+    memcpy(self->mb_data + left, src_view.buf, count);
+    PyBuffer_Release(&src_view);
     return 0;
 }
 
@@ -232,6 +237,14 @@
 
 #if PY_MAJOR_VERSION >= 3
 /* pfffffffffffff pages of copy-paste from listobject.c */
+
+/* pfffffffffffff#2: the PySlice_GetIndicesEx() *macro* should not
+   be called, because C extension modules compiled with it differ
+   on ABI between 3.6.0, 3.6.1 and 3.6.2. */
+#if PY_VERSION_HEX < 0x03070000 && defined(PySlice_GetIndicesEx) && !defined(PYPY_VERSION)
+#undef PySlice_GetIndicesEx
+#endif
+
 static PyObject *mb_subscript(MiniBufferObj *self, PyObject *item)
 {
     if (PyIndex_Check(item)) {
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
 # The short X.Y version.
 version = '1.11'
 # The full version, including alpha/beta/rc tags.
-release = '1.11.0'
+release = '1.11.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,13 +51,13 @@
 
 Download and Installation:
 
-* https://pypi.python.org/packages/4e/32/4070bdf32812c89eb635c80880a5caa2e0189aa7999994c265577e5154f3/cffi-1.11.0.tar.gz#md5=2c5939cc2fa0183fe0c2fcb6a4f1994d
+* https://pypi.python.org/pypi/cffi
 
-   - MD5: 2c5939cc2fa0183fe0c2fcb6a4f1994d
+   - MD5: ...
 
-   - SHA: 93cb5aaf152e19f9d4082a723aa2396e9cd5d93f
+   - SHA: ...
 
-   - SHA256: 5f4ff33371c6969b39b293d9771ee91e81d26f9129be093ca1b7be357fcefd15
+   - SHA256: ...
 
 * Or grab the most current version from the `Bitbucket page`_:
   ``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -3,6 +3,17 @@
 ======================
 
 
+v1.11.1
+=======
+
+* Fix tests, remove deprecated C API usage
+
+* Fix for 3.6.0/3.6.1 being incompatible with each other unless we hack
+  (cpython issue `#29943`_)
+  
+.. _`#29943`: https://bugs.python.org/issue29943
+
+
 v1.11
 =====
 
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -186,7 +186,7 @@
 
 `Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
 """,
-        version='1.11.0',
+        version='1.11.1',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
                                '_embedding.h', '_cffi_errors.h']}
@@ -231,6 +231,7 @@
             'Programming Language :: Python :: 3.3',
             'Programming Language :: Python :: 3.4',
             'Programming Language :: Python :: 3.5',
+            'Programming Language :: Python :: 3.6',
             'Programming Language :: Python :: Implementation :: CPython',
             'Programming Language :: Python :: Implementation :: PyPy',
         ],
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -2455,9 +2455,18 @@
     pt = lib.call2(lib.cb2)
     assert (pt.x, pt.y) == (99*500*999, -99*500*999)
 
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
 def test_ffi_gc_size_arg():
     # with PyPy's GC, these calls to ffi.gc() would rapidly consume
     # 40 GB of RAM without the third argument
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2466,8 +2475,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
 
@@ -2477,6 +2486,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2487,8 +2497,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2506,8 +2516,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x
diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py
--- a/testing/cffi0/test_version.py
+++ b/testing/cffi0/test_version.py
@@ -36,7 +36,7 @@
     v = cffi.__version__.replace('+', '')
     p = os.path.join(parent, 'doc', 'source', 'installation.rst')
     content = open(p).read()
-    assert ("cffi/cffi-%s.tar.gz" % v) in content
+    assert ("/cffi-%s.tar.gz" % v) in content
 
 def test_setup_version():
     parent = os.path.dirname(os.path.dirname(cffi.__file__))
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -2294,7 +2294,16 @@
     assert ffi.typeof("UINT_PTR") is ffi.typeof(expected)
     assert ffi.typeof("PTSTR") is ffi.typeof("wchar_t *")
 
-def test_gc_pypy_size_arg():
+def _only_test_on_linux_intel():
+    if not sys.platform.startswith('linux'):
+        py.test.skip('only running the memory-intensive test on Linux')
+    import platform
+    machine = platform.machine()
+    if 'x86' not in machine and 'x64' not in machine:
+        py.test.skip('only running the memory-intensive test on x86/x64')
+
+def test_ffi_gc_size_arg():
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2303,8 +2312,8 @@
     for i in range(2000):
         p = lib.malloc(20*1024*1024)    # 20 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 20*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 20*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 20*1024*1024)
         del p
         # with PyPy's GC, the above would rapidly consume 40 GB of RAM
@@ -2316,6 +2325,7 @@
     # is skipped on CPython, where it eats all the memory.
     if '__pypy__' not in sys.builtin_module_names:
         py.test.skip("find a way to tweak the cyclic GC of CPython")
+    _only_test_on_linux_intel()
     ffi = FFI()
     ffi.cdef("void *malloc(size_t); void free(void *);")
     lib = ffi.verify(r"""
@@ -2326,8 +2336,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2345,8 +2355,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x

From pypy.commits at gmail.com  Wed Sep 27 07:12:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 04:12:23 -0700 (PDT)
Subject: [pypy-commit] cffi default: More fun with _PyThreadState_Current
 becoming undefined in 3.7.
Message-ID: <59cb8797.133f1c0a.b4b21.6c1f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3029:989d98a58c95
Date: 2017-09-27 13:11 +0200
http://bitbucket.org/cffi/cffi/changeset/989d98a58c95/

Log:	More fun with _PyThreadState_Current becoming undefined in 3.7.

diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -65,22 +65,30 @@
 /* Seems that CPython 3.5.1 made our job harder.  Did not find out how
    to do that without these hacks.  We can't use PyThreadState_GET(),
    because that calls PyThreadState_Get() which fails an assert if the
-   result is NULL. */
-#if PY_MAJOR_VERSION >= 3 && !defined(_Py_atomic_load_relaxed)
-                             /* this was abruptly un-defined in 3.5.1 */
-void *volatile _PyThreadState_Current;
-   /* XXX simple volatile access is assumed atomic */
-#  define _Py_atomic_load_relaxed(pp)  (*(pp))
-#endif
-
+   result is NULL.  We can use _PyThreadState_UncheckedGet() from 3.6,
+   though.  It was added in 3.5.2 but should never be used in 3.5.x
+   because it is not available in 3.5.0 or 3.5.1. */
+#if PY_VERSION_HEX >= 0x03060000
 static PyThreadState *get_current_ts(void)
 {
-#if defined(_Py_atomic_load_relaxed)
+    return _PyThreadState_UncheckedGet();
+}
+#else
+#  if PY_MAJOR_VERSION >= 3 && !defined(_Py_atomic_load_relaxed)
+                             /* this was abruptly un-defined in 3.5.1 */
+extern void *volatile _PyThreadState_Current;
+   /* XXX simple volatile access is assumed atomic */
+#    define _Py_atomic_load_relaxed(pp)  (*(pp))
+#  endif
+static PyThreadState *get_current_ts(void)
+{
+#  if defined(_Py_atomic_load_relaxed)
     return (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current);
-#else
+#  else
     return _PyThreadState_Current;
+#  endif
+}
 #endif
-}
 
 static PyGILState_STATE gil_ensure(void)
 {

From pypy.commits at gmail.com  Wed Sep 27 07:12:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 04:12:25 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.11: hg merge default
Message-ID: <59cb8799.8fb6df0a.4d8f.46cd@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.11
Changeset: r3030:aadf924dcf23
Date: 2017-09-27 13:12 +0200
http://bitbucket.org/cffi/cffi/changeset/aadf924dcf23/

Log:	hg merge default

diff --git a/c/misc_thread_common.h b/c/misc_thread_common.h
--- a/c/misc_thread_common.h
+++ b/c/misc_thread_common.h
@@ -65,22 +65,30 @@
 /* Seems that CPython 3.5.1 made our job harder.  Did not find out how
    to do that without these hacks.  We can't use PyThreadState_GET(),
    because that calls PyThreadState_Get() which fails an assert if the
-   result is NULL. */
-#if PY_MAJOR_VERSION >= 3 && !defined(_Py_atomic_load_relaxed)
-                             /* this was abruptly un-defined in 3.5.1 */
-void *volatile _PyThreadState_Current;
-   /* XXX simple volatile access is assumed atomic */
-#  define _Py_atomic_load_relaxed(pp)  (*(pp))
-#endif
-
+   result is NULL.  We can use _PyThreadState_UncheckedGet() from 3.6,
+   though.  It was added in 3.5.2 but should never be used in 3.5.x
+   because it is not available in 3.5.0 or 3.5.1. */
+#if PY_VERSION_HEX >= 0x03060000
 static PyThreadState *get_current_ts(void)
 {
-#if defined(_Py_atomic_load_relaxed)
+    return _PyThreadState_UncheckedGet();
+}
+#else
+#  if PY_MAJOR_VERSION >= 3 && !defined(_Py_atomic_load_relaxed)
+                             /* this was abruptly un-defined in 3.5.1 */
+extern void *volatile _PyThreadState_Current;
+   /* XXX simple volatile access is assumed atomic */
+#    define _Py_atomic_load_relaxed(pp)  (*(pp))
+#  endif
+static PyThreadState *get_current_ts(void)
+{
+#  if defined(_Py_atomic_load_relaxed)
     return (PyThreadState*)_Py_atomic_load_relaxed(&_PyThreadState_Current);
-#else
+#  else
     return _PyThreadState_Current;
+#  endif
+}
 #endif
-}
 
 static PyGILState_STATE gil_ensure(void)
 {

From pypy.commits at gmail.com  Wed Sep 27 07:25:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 04:25:59 -0700 (PDT)
Subject: [pypy-commit] cffi default: Mention the fix 989d98a58c95
Message-ID: <59cb8ac7.05cf1c0a.23887.801a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3031:7005b3b4e54f
Date: 2017-09-27 13:25 +0200
http://bitbucket.org/cffi/cffi/changeset/7005b3b4e54f/

Log:	Mention the fix 989d98a58c95

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -10,6 +10,8 @@
 
 * Fix for 3.6.0/3.6.1 being incompatible with each other unless we hack
   (cpython issue `#29943`_)
+
+* Fix for 3.7.0a1+ (it didn't run successfully in CPython's 3.7 branch)
   
 .. _`#29943`: https://bugs.python.org/issue29943
 

From pypy.commits at gmail.com  Wed Sep 27 07:26:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 04:26:01 -0700 (PDT)
Subject: [pypy-commit] cffi release-1.11: hg merge default
Message-ID: <59cb8ac9.8298df0a.b5272.1d5f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-1.11
Changeset: r3032:d3e081232b0b
Date: 2017-09-27 13:25 +0200
http://bitbucket.org/cffi/cffi/changeset/d3e081232b0b/

Log:	hg merge default

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -10,6 +10,8 @@
 
 * Fix for 3.6.0/3.6.1 being incompatible with each other unless we hack
   (cpython issue `#29943`_)
+
+* Fix for 3.7.0a1+ (it didn't run successfully in CPython's 3.7 branch)
   
 .. _`#29943`: https://bugs.python.org/issue29943
 

From pypy.commits at gmail.com  Wed Sep 27 08:28:54 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Wed, 27 Sep 2017 05:28:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Try to fix translation on Windows: skip
 socket.sendmsg and associated macros.
Message-ID: <59cb9986.d1141c0a.e6562.e898@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r92475:826d99189c3b
Date: 2017-09-27 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/826d99189c3b/

Log:	Try to fix translation on Windows: skip socket.sendmsg and
	associated macros.

diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -37,7 +37,8 @@
             CMSG_SPACE CMSG_LEN
             """.split():
 
-            if (name in ('inet_pton', 'inet_ntop', 'socketpair') and
+            if (name in ('inet_pton', 'inet_ntop', 'socketpair',
+                         'CMSG_SPACE', 'CMSG_LEN') and
                 not hasattr(rsocket, name)):
                 continue
 
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -879,11 +879,14 @@
 socketmethodnames = """
 _accept bind close connect connect_ex fileno detach
 getpeername getsockname getsockopt gettimeout listen
-recv recvfrom recvmsg send sendall sendto sendmsg setblocking
+recv recvfrom send sendall sendto setblocking
 setsockopt settimeout shutdown _reuse _drop recv_into recvfrom_into
 """.split()
 if hasattr(rsocket._c, 'WSAIoctl'):
     socketmethodnames.append('ioctl')
+if rsocket._c.HAVE_SENDMSG:
+    socketmethodnames.append('sendmsg')
+    socketmethodnames.append('recvmsg')
 
 socketmethods = {}
 for methodname in socketmethodnames:
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -348,7 +348,8 @@
                                  ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 # insert handler for sendmsg / recvmsg here
-if _POSIX:
+HAVE_SENDMSG = bool(_POSIX)
+if HAVE_SENDMSG:
     includes = ['stddef.h',
                 'sys/socket.h',
                 'unistd.h',
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1393,7 +1393,7 @@
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
-if _c._POSIX:
+if _c.HAVE_SENDMSG:
     def CMSG_LEN( demanded_len):
         """
         Socket method to determine the optimal byte size of the ancillary.

From pypy.commits at gmail.com  Wed Sep 27 10:03:26 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 27 Sep 2017 07:03:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: (fijal, arigo) change 10M to 10k,
 we have no clue what that number should be
Message-ID: <59cbafae.952d1c0a.b475e.1254@mx.google.com>

Author: fijal
Branch: 
Changeset: r92476:b67bc051d0c1
Date: 2017-09-27 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/b67bc051d0c1/

Log:	(fijal, arigo) change 10M to 10k, we have no clue what that number
	should be

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1315,7 +1315,7 @@
         if not ctx:
             raise ssl_error(space, "failed to allocate SSL context")
 
-        rgc.add_memory_pressure(10 * 1024 * 1024)
+        rgc.add_memory_pressure(10 * 1024)
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
         self.check_hostname = False

From pypy.commits at gmail.com  Wed Sep 27 11:32:32 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Sep 2017 08:32:32 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: merge default
Message-ID: <59cbc490.b5b8df0a.aec9c.f55e@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92477:ad2fa98841d9
Date: 2017-09-27 16:05 +0200
http://bitbucket.org/pypy/pypy/changeset/ad2fa98841d9/

Log:	merge default

diff too long, truncating to 2000 out of 6770 lines

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -364,7 +364,7 @@
                 pypy_dll = _ffi.CDLL(name, mode)
             else:
                 pypy_dll = _ffi.WinDLL(name, mode)
-            self._pypy_dll = pypy_dll
+            self.__pypy_dll__ = pypy_dll
             handle = int(pypy_dll)
             if _sys.maxint > 2 ** 32:
                 handle = int(handle)   # long -> int
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except (AttributeError, StopIteration):
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -82,7 +82,7 @@
         return False
 
     def in_dll(self, dll, name):
-        return self.from_address(dll._pypy_dll.getaddressindll(name))
+        return self.from_address(dll.__pypy_dll__.getaddressindll(name))
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -430,7 +430,7 @@
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
 
-        cdll = self.dll._pypy_dll
+        cdll = self.dll.__pypy_dll__
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffi_restype = restype.get_ffi_argtype()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -142,6 +142,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = buffer(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FFL) << 56) | \
+               ((value & 0x000000000000FF00L) << 40) | \
+               ((value & 0x0000000000FF0000L) << 24) | \
+               ((value & 0x00000000FF000000L) << 8) | \
+               ((value & 0x000000FF00000000L) >> 8) | \
+               ((value & 0x0000FF0000000000L) >> 24) | \
+               ((value & 0x00FF000000000000L) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -271,6 +319,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -146,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -240,19 +241,27 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData):
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -116,10 +116,20 @@
 
 @builtinify
 def dump(obj, file, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     Pickler(file, protocol).dump(obj)
 
 @builtinify
 def dumps(obj, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     file = StringIO()
     Pickler(file, protocol).dump(obj)
     return file.getvalue()
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -214,7 +214,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -119,8 +119,15 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian and Ubuntu, this is the command to install all build-time
-dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -27,8 +27,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -69,6 +69,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -77,14 +78,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -123,6 +123,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -149,11 +150,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -161,6 +164,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -171,6 +175,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -181,14 +186,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -210,7 +213,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -242,6 +244,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -272,6 +275,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -292,12 +296,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -429,7 +429,8 @@
 
 * the ``__builtins__`` name is always referencing the ``__builtin__`` module,
   never a dictionary as it sometimes is in CPython. Assigning to
-  ``__builtins__`` has no effect.
+  ``__builtins__`` has no effect.  (For usages of tools like
+  RestrictedPython, see `issue #2653`_.)
 
 * directly calling the internal magic methods of a few built-in types
   with invalid arguments may have a slightly different result.  For
@@ -535,7 +536,12 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
+* In PyPy, module and class dictionaries are optimized under the assumption
+  that deleting attributes from them are rare. Because of this, e.g.
+  ``del foo.bar`` where ``foo`` is a module (or class) that contains the
+  function ``bar``, is significantly slower than CPython.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
-
+.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.9.0.rst
    release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.9.0.rst
    whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
@@ -36,6 +37,7 @@
 .. toctree::
 
    whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
 
diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.9.0.rst
@@ -0,0 +1,209 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.9 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 includes the upstream stdlib version 3.5.3.
+
+Only a handful of failing tests remain in NumPy and Pandas on PyPy2.7, issues
+that appeared as excessive memory use were cleared up and other incompatibilities
+were resolved.
+
+Cython 0.27 (released last week) should support more projects with PyPy, both
+on PyPy2.7 and PyPy3.5 beta.
+
+We optimized the JSON parser for recurring string keys, which should decrease
+memory use to 50% and increase parsing speed by up to 15% for large JSON files
+with many repeating dictionary keys (which is quite common).
+
+CFFI_, which is part of the PyPy release, has been updated to 1.11,
+improving an already great package for interfacing with C. CFFI now supports
+complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has
+improved support for callbacks.
+
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy. Of course the bug fixes and performance enhancements
+mentioned above are part of both PyPy2.7 and PyPy3.5 beta.
+
+As always, this release fixed many other issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.9 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _vmprof: http://vmprof.readthedocs.io
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017)
+======================================================================================
+
+See also issues that were resolved_
+
+Note that these are also merged into PyPy 3.5
+
+* New features and cleanups
+
+  * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``,
+    ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``,
+    ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat``
+  * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible
+  * ``gcdump`` now reports largest object
+  * More complete support in the ``_curses`` CFFI module
+  * Add cPickle.Unpickler.find_global (issue 1853_)
+  * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set
+  * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()``
+  * Update RevDB to version pypy2.7-v5.6.2
+  * Previously, ``instance.method`` would return always the same bound method
+    object, when gotten from the same instance (as far as ``is`` and ``id()``
+    can tell).  CPython doesn't do that.  Now PyPy, like CPython, returns a 
+    different bound method object every time.  For ``type.method``, PyPy2 still
+    returns always the same *unbound* method object; CPython does it for built-in
+    types but not for user-defined types
+  * Link to disable PaX protection for the JIT when needed
+  * Update build instructions and an rarely used Makefile
+  * Recreate support for using leakfinder in cpyext tests which had suffered
+    bit-rot, disable due to many false positives
+  * Add more functionality to ``sysconfig``
+  * Added ``_swappedbytes_`` support for ``ctypes.Structure``
+  * Better support the ``inspect`` module on ``frames``
+
+* Bug Fixes 
+
+  * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
+  * Implement ``PyListOjbect.getstorage_copy``
+  * Fix for ``reversed(dictproxy)`` issue 2601_
+  * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_
+  * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
+  * ``gc.get_objects`` now handles objects with finalizers more consistently
+  * Fixed memory leak in ``SSLContext.getpeercert`` returning validated
+    certificates and ``SSLContext.get_ca_certs(binary_mode=True)``
+    (_get_crl_dp) `CPython issue 29738`_
+
+* Performance improvements:
+
+  * Improve performance of ``bytearray.extend`` by rewriting portions in app-level
+  * Optimize list accesses with constant indexes better by retaining more
+    information about them
+  * Add a jit driver for ``array.count`` and ``array.index``
+  * Improve information retained in a bridge wrt ``array``
+  * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into
+    pure C macros
+  * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
+    all the ints
+  * Cache string keys that occur in JSON dicts, as they are likely to repeat
+
+* RPython improvements
+
+  * Do not preallocate a RPython list if we only know an upper bound on its size
+  * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
+  * Replace magical NOT RPYTHON comment with a decorator
+  * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
+
+* Degredations
+
+  * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module
+
+.. _here: cpython_differences.html
+.. _1853: https://bitbucket.org/pypy/pypy/issues/1853
+.. _2592: https://bitbucket.org/pypy/pypy/issues/2592
+.. _2590: https://bitbucket.org/pypy/pypy/issues/2590
+.. _2621: https://bitbucket.org/pypy/pypy/issues/2621
+
+Highlights of the PyPy3.5 release (since 5.8 beta released June 2017)
+======================================================================
+
+* New features
+
+  * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
+    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
+    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``
+  * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
+  * Support the new version of ``os.stat()`` on win32
+  * Use ``stat3()`` on Posix
+  * Accept buffer objects as filenames, except for `oslistdir``
+  * Make slices of array ``memoryview`` s usable as writable buffers if contiguous
+  * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded
+  * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version
+  * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence
+  * Enable PGO for CLang
+  * Rework ``cppyy`` packaging and rename the backend to ``_cppyy``
+  * Support for libressl 2.5.4
+  * Mirror CPython ``classmethod __reduce__`` which fixes pickling test
+  * Use utf-8 for ``readline`` history file
+  * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses
+  * Add async slot functions in cpyext
+
+* Bug Fixes
+
+  * Try to make ``openssl`` CFFI bindings more general and future-proof
+  * Better support ``importlib`` by only listing built-in modules in ``sys.builtin``
+  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_
+  * Fix for ``reversed(mapping object)`` issue 2601_
+  * Fixing regression with non-started generator receiving non-``None``, should
+    always raise ``TypeError``
+  * ``itertools.islice``: use same logic as CPython, fixes 2643_
+
+* Performance improvements:
+
+  * 
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+
+.. _resolved: whatsnew-pypy2-5.9.0.html
+.. _2579: https://bitbucket.org/pypy/pypy/issues/2579
+.. _2601: https://bitbucket.org/pypy/pypy/issues/2601
+.. _2643: https://bitbucket.org/pypy/pypy/issues/2643
+.. _CPython issue 29738: https://bugs.python.org/issue29738
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,75 +1,6 @@
-==========================
-What's new in PyPy2.7 5.9+
-==========================
-
-.. this is a revision shortly after release-pypy2.7-v5.8.0
-.. startrev: 558bd00b3dd8
-
-In previous versions of PyPy, ``instance.method`` would return always
-the same bound method object, when gotten out of the same instance (as
-far as ``is`` and ``id()`` can tell).  CPython doesn't do that.  Now
-PyPy, like CPython, returns a different bound method object every time.
-For ``type.method``, PyPy2 still returns always the same *unbound*
-method object; CPython does it for built-in types but not for
-user-defined types.
-
-.. branch: cffi-complex
-.. branch: cffi-char16-char32
-
-The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
-
-.. branch: ctypes_char_indexing
-
-Indexing into char* behaves differently than CPython
-
-.. branch: vmprof-0.4.8
-
-Improve and fix issues with vmprof
-
-.. branch: issue-2592
-
-CPyext PyListObject.pop must return the value
-
-.. branch: cpyext-hash_notimpl
-
-If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None
-
-.. branch: cppyy-packaging
-
-Renaming of ``cppyy`` to ``_cppyy``.
-The former is now an external package installable with ``pip install cppyy``.
-
-.. branch: Enable_PGO_for_clang
-
-.. branch: nopax
-
-At the end of translation, run ``attr -q -s pax.flags -V m`` on
-PAX-enabled systems on the produced binary.  This seems necessary
-because PyPy uses a JIT.
-
-.. branch: pypy_bytearray
-
-Improve ``bytearray`` performance (backported from py3.5)
-
-.. branch: gc-del-limit-growth
-
-Fix the bounds in the GC when allocating a lot of objects with finalizers,
-fixes issue #2590
-
-.. branch: arrays-force-less
-
-Small improvement to optimize list accesses with constant indexes better by
-throwing away information about them less eagerly.
-
-
-.. branch: getarrayitem-into-bridges
-
-More information is retained into a bridge: knowledge about the content of
-arrays (at fixed indices) is stored in guards (and thus available at the
-beginning of bridges). Also, some better feeding of information about known
-fields of constant objects into bridges.
-
-.. branch: cpyext-leakchecking
-
-Add support for leakfinder in cpyext tests (disabled for now, due to too many
-failures).
+===========================
+What's new in PyPy2.7 5.10+
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.9.0
+.. startrev:899e5245de1e
diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst
@@ -0,0 +1,87 @@
+=========================
+What's new in PyPy2.7 5.9
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.8.0
+.. startrev: 558bd00b3dd8
+
+In previous versions of PyPy, ``instance.method`` would return always
+the same bound method object, when gotten out of the same instance (as
+far as ``is`` and ``id()`` can tell).  CPython doesn't do that.  Now
+PyPy, like CPython, returns a different bound method object every time.
+For ``type.method``, PyPy2 still returns always the same *unbound*
+method object; CPython does it for built-in types but not for
+user-defined types.
+
+.. branch: cffi-complex
+.. branch: cffi-char16-char32
+
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
+
+.. branch: ctypes_char_indexing
+
+Indexing into char* behaves differently than CPython
+
+.. branch: vmprof-0.4.8
+
+Improve and fix issues with vmprof
+
+.. branch: issue-2592
+
+CPyext PyListObject.pop must return the value
+
+.. branch: cpyext-hash_notimpl
+
+If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None
+
+.. branch: cppyy-packaging
+
+Renaming of ``cppyy`` to ``_cppyy``.
+The former is now an external package installable with ``pip install cppyy``.
+
+.. branch: Enable_PGO_for_clang
+
+.. branch: nopax
+
+At the end of translation, run ``attr -q -s pax.flags -V m`` on
+PAX-enabled systems on the produced binary.  This seems necessary
+because PyPy uses a JIT.
+
+.. branch: pypy_bytearray
+
+Improve ``bytearray`` performance (backported from py3.5)
+
+.. branch: gc-del-limit-growth
+
+Fix the bounds in the GC when allocating a lot of objects with finalizers,
+fixes issue #2590
+
+.. branch: arrays-force-less
+
+Small improvement to optimize list accesses with constant indexes better by
+throwing away information about them less eagerly.
+
+
+.. branch: getarrayitem-into-bridges
+
+More information is retained into a bridge: knowledge about the content of
+arrays (at fixed indices) is stored in guards (and thus available at the
+beginning of bridges). Also, some better feeding of information about known
+fields of constant objects into bridges.
+
+.. branch: cpyext-leakchecking
+
+Add support for leakfinder in cpyext tests (disabled for now, due to too many
+failures).
+
+.. branch: pypy_swappedbytes
+
+Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.
+
+.. branch: py_ssize_t
+
+Explicitly use Py_ssize_t as the Signed type in pypy c-api
diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst
@@ -0,0 +1,7 @@
+=======================
+What's new in PyPy3 5.9
+=======================
+
+.. this is the revision after release-pypy3.5-5.8
+.. startrev: afbf09453369
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
 =========================
-What's new in PyPy3 5.7+
+What's new in PyPy3 5.9+
 =========================
 
-.. this is the revision after release-pypy3.3-5.7.x was branched
-.. startrev: afbf09453369
+.. this is the revision after release-pypy3.5-5.9
+.. startrev: be41e3ac0a29
 
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -114,12 +114,15 @@
 INCLUDE, LIB and PATH (for DLLs) environment variables appropriately.
 
 
-Abridged method (for -Ojit builds using Visual Studio 2008)
------------------------------------------------------------
+Abridged method (using Visual Studio 2008)
+------------------------------------------
 
 Download the versions of all the external packages from
+https://bitbucket.org/pypy/pypy/downloads/local_59.zip
+(for post-5.8 builds) with sha256 checksum
+``6344230e90ab7a9cb84efbae1ba22051cdeeb40a31823e0808545b705aba8911``
 https://bitbucket.org/pypy/pypy/downloads/local_5.8.zip
-(for post-5.7.1 builds) with sha256 checksum 
+(to reproduce 5.8 builds) with sha256 checksum 
 ``fbe769bf3a4ab6f5a8b0a05b61930fc7f37da2a9a85a8f609cf5a9bad06e2554`` or
 https://bitbucket.org/pypy/pypy/downloads/local_2.4.zip
 (for 2.4 release and later) or
@@ -135,8 +138,8 @@
 Now you should be good to go. If you choose this method, you do not need
 to download/build anything else. 
 
-Nonabrided method (building from scratch)
------------------------------------------
+Nonabridged method (building from scratch)
+------------------------------------------
 
 If you want to, you can rebuild everything from scratch by continuing.
 
@@ -209,17 +212,85 @@
 The expat XML parser
 ~~~~~~~~~~~~~~~~~~~~
 
-Download the source code of expat on sourceforge:
-https://github.com/libexpat/libexpat/releases and extract it in the base directory.
-Version 2.1.1 is known to pass tests. Then open the project file ``expat.dsw``
-with Visual Studio; follow the instruction for converting the project files,
-switch to the "Release" configuration, use the ``expat_static`` project,
-reconfigure the runtime for Multi-threaded DLL (/MD) and build. Do the same for
-the ``expat`` project to build the ``expat.dll`` (for tests via ll2ctypes)
+CPython compiles expat from source as part of the build. PyPy uses the same
+code base, but expects to link to a static lib of expat. Here are instructions
+to reproduce the static lib in version 2.2.4.
 
-Then, copy the file ``win32\bin\release\libexpat.lib`` into
-LIB, and both ``lib\expat.h`` and ``lib\expat_external.h`` in
-INCLUDE, and ``win32\bin\release\libexpat.dll`` into PATH.
+Download the source code of expat: https://github.com/libexpat/libexpat. 
+``git checkout`` the proper tag, in this case ``R_2_2_4``. Run
+``vcvars.bat`` to set up the visual compiler tools, and CD into the source
+directory. Create a file ``stdbool.h`` with the content
+
+.. code-block:: c
+
+    #pragma once
+
+    #define false   0
+    #define true    1
+
+    #define bool int
+
+and put it in a place on the ``INCLUDE`` path, or create it in the local
+directory and add ``.`` to the ``INCLUDE`` path::
+
+    SET INCLUDE=%INCLUDE%;.
+
+Then compile all the ``*.c`` file into ``*.obj``::
+
+    cl.exe /nologo /MD  /O2 *c /c
+    rem for debug
+    cl.exe /nologo /MD  /O0 /Ob0 /Zi *c /c
+
+You may need to move some variable declarations to the beginning of the
+function, to be compliant with C89 standard. Here is the diff for version 2.2.4
+
+.. code-block:: diff
+
+    diff --git a/expat/lib/xmltok.c b/expat/lib/xmltok.c
+    index 007aed0..a2dcaad 100644
+    --- a/expat/lib/xmltok.c
+    +++ b/expat/lib/xmltok.c
+    @@ -399,19 +399,21 @@ utf8_toUtf8(const ENCODING *UNUSED_P(enc),
+       /* Avoid copying partial characters (due to limited space). */
+       const ptrdiff_t bytesAvailable = fromLim - *fromP;
+       const ptrdiff_t bytesStorable = toLim - *toP;
+    +  const char * fromLimBefore;
+    +  ptrdiff_t bytesToCopy;
+       if (bytesAvailable > bytesStorable) {
+         fromLim = *fromP + bytesStorable;
+         output_exhausted = true;
+       }
+
+       /* Avoid copying partial characters (from incomplete input). */
+    -  const char * const fromLimBefore = fromLim;
+    +  fromLimBefore = fromLim;
+       align_limit_to_full_utf8_characters(*fromP, &fromLim);
+       if (fromLim < fromLimBefore) {
+         input_incomplete = true;
+       }
+
+    -  const ptrdiff_t bytesToCopy = fromLim - *fromP;
+    +  bytesToCopy = fromLim - *fromP;
+       memcpy((void *)*toP, (const void *)*fromP, (size_t)bytesToCopy);
+       *fromP += bytesToCopy;
+       *toP += bytesToCopy;
+
+
+Create ``libexpat.lib`` (for translation) and ``libexpat.dll`` (for tests)::
+
+    cl /LD *.obj libexpat.def /Felibexpat.dll 
+    rem for debug
+    rem cl /LDd /Zi *.obj libexpat.def /Felibexpat.dll
+
+    rem this will override the export library created in the step above
+    rem but tests do not need the export library, they load the dll dynamically
+    lib *.obj /out:libexpat.lib
+
+Then, copy 
+
+- ``libexpat.lib`` into LIB
+- both ``lib\expat.h`` and ``lib\expat_external.h`` in INCLUDE
+- ``libexpat.dll`` into PATH
 
 
 The OpenSSL library
@@ -363,7 +434,7 @@
 It is probably not too much work if the goal is only to get a translated
 PyPy executable, and to run all tests before translation.  But you need
 to start somewhere, and you should start with some tests in
-rpython/translator/c/test/, like ``test_standalone.py`` and
+``rpython/translator/c/test/``, like ``test_standalone.py`` and
 ``test_newgc.py``: try to have them pass on top of CPython64/64.
 
 Keep in mind that this runs small translations, and some details may go
@@ -373,7 +444,7 @@
 should be something like ``long long``.
 
 What is more generally needed is to review all the C files in
-rpython/translator/c/src for the word ``long``, because this means a
+``rpython/translator/c/src`` for the word ``long``, because this means a
 32-bit integer even on Win64.  Replace it with ``Signed`` most of the
 times.  You can replace one with the other without breaking anything on
 any other platform, so feel free to.
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1442,7 +1442,7 @@
             length = 1
         return start, stop, step, length
 
-    def getindex_w(self, w_obj, w_exception, objdescr=None):
+    def getindex_w(self, w_obj, w_exception, objdescr=None, errmsg=None):
         """Return w_obj.__index__() as an RPython int.
         If w_exception is None, silently clamp in case of overflow;
         else raise w_exception.
@@ -1452,8 +1452,10 @@
         except OperationError as err:
             if objdescr is None or not err.match(self, self.w_TypeError):
                 raise
-            raise oefmt(self.w_TypeError, "%s must be an integer, not %T",
-                        objdescr, w_obj)
+            if errmsg is None:
+                errmsg = " must be an integer"
+            raise oefmt(self.w_TypeError, "%s%s, not %T",
+                        objdescr, errmsg, w_obj)
         try:
             # allow_conversion=False it's not really necessary because the
             # return type of __index__ is already checked by space.index(),
@@ -1629,12 +1631,15 @@
         # return text_w(w_obj) or None
         return None if self.is_none(w_obj) else self.text_w(w_obj)
 
+    @specialize.argtype(1)
     def bytes_w(self, w_obj):
         """ Takes an application level :py:class:`bytes`
             (on PyPy2 this equals `str`) and returns a rpython byte string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
+    @specialize.argtype(1)
     def text_w(self, w_obj):
         """ PyPy2 takes either a :py:class:`str` and returns a
             rpython byte string, or it takes an :py:class:`unicode`
@@ -1644,6 +1649,7 @@
             On PyPy3 it takes a :py:class:`str` and it will return
             an utf-8 encoded rpython string.
         """
+        assert w_obj is not None
         return w_obj.str_w(self)
 
     @not_rpython    # tests only; should be replaced with bytes_w or text_w
@@ -1686,12 +1692,14 @@
             if len(string) != 1:
                 raise oefmt(self.w_ValueError, "string must be of size 1")
             return string[0]
-        value = self.getindex_w(w_obj, None)
+        value = self.getindex_w(w_obj, None, "",
+                                "an integer or string of size 1 is required")
         if not 0 <= value < 256:
             # this includes the OverflowError in case the long is too large
             raise oefmt(self.w_ValueError, "byte must be in range(0, 256)")
         return chr(value)
 
+    @specialize.argtype(1)
     def int_w(self, w_obj, allow_conversion=True):
         """
         Unwrap an app-level int object into an interpret-level int.
@@ -1704,26 +1712,35 @@
         If allow_conversion=False, w_obj needs to be an app-level int or a
         subclass.
         """
+        assert w_obj is not None
         return w_obj.int_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def int(self, w_obj):
+        assert w_obj is not None
         return w_obj.int(self)
 
+    @specialize.argtype(1)
     def uint_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.uint_w(self)
 
+    @specialize.argtype(1)
     def bigint_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return a rlib.rbigint object and call __long__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.bigint_w(self, allow_conversion)
 
+    @specialize.argtype(1)
     def float_w(self, w_obj, allow_conversion=True):
         """
         Like int_w, but return an interp-level float and call __float__ if
         allow_conversion is True.
         """
+        assert w_obj is not None
         return w_obj.float_w(self, allow_conversion)
 
     def realtext_w(self, w_obj):
@@ -1733,7 +1750,9 @@
             raise oefmt(self.w_TypeError, "argument must be a string")
         return self.bytes_w(w_obj)
 
+    @specialize.argtype(1)
     def unicode_w(self, w_obj):
+        assert w_obj is not None
         return w_obj.unicode_w(self)
 
     def unicode0_w(self, w_obj):
@@ -1758,7 +1777,9 @@
         # This is here mostly just for gateway.int_unwrapping_space_method().
         return bool(self.int_w(w_obj))
 
+    @specialize.argtype(1)
     def ord(self, w_obj):
+        assert w_obj is not None
         return w_obj.ord(self)
 
     # This is all interface for gateway.py.
diff --git a/pypy/interpreter/test/test_typedef.py b/pypy/interpreter/test/test_typedef.py
--- a/pypy/interpreter/test/test_typedef.py
+++ b/pypy/interpreter/test/test_typedef.py
@@ -419,3 +419,7 @@
         def f():
             return x
         assert f.__closure__[0].cell_contents is x
+
+    def test_get_with_none_arg(self):
+        raises(TypeError, type.__dict__['__mro__'].__get__, None)
+        raises(TypeError, type.__dict__['__mro__'].__get__, None, None)
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -297,6 +297,8 @@
         if (space.is_w(w_obj, space.w_None)
             and not space.is_w(w_cls, space.type(space.w_None))):
             #print self, w_obj, w_cls
+            if space.is_w(w_cls, space.w_None):
+                raise oefmt(space.w_TypeError, "__get__(None, None) is invalid")
             return self
         else:
             try:
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.11.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py
--- a/pypy/module/_csv/interp_csv.py
+++ b/pypy/module/_csv/interp_csv.py
@@ -29,10 +29,15 @@
         return default
     return space.is_true(w_src)
 
-def _get_int(space, w_src, default):
+def _get_int(space, w_src, default, attrname):
     if w_src is None:
         return default
-    return space.int_w(w_src)
+    try:
+        return space.int_w(w_src)
+    except OperationError as e:
+        if e.match(space, space.w_TypeError):
+            raise oefmt(space.w_TypeError, '"%s" must be an int', attrname)
+        raise
 
 def _get_str(space, w_src, default, attrname):
     if w_src is None:
@@ -100,7 +105,7 @@
     dialect.escapechar = _get_char(space, w_escapechar, '\0', 'escapechar')
     dialect.lineterminator = _get_str(space, w_lineterminator, '\r\n', 'lineterminator')
     dialect.quotechar = _get_char(space, w_quotechar, '"', 'quotechar')
-    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL)
+    tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL, 'quoting')
     dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False)
     dialect.strict = _get_bool(space, w_strict, False)
 
diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py
--- a/pypy/module/_csv/test/test_dialect.py
+++ b/pypy/module/_csv/test/test_dialect.py
@@ -65,7 +65,8 @@
             name = attempt[0]
             for value in attempt[1:]:
                 kwargs = {name: value}
-                raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                exc_info = raises(TypeError, _csv.register_dialect, 'foo1', **kwargs)
+                assert name in exc_info.value.args[0]
 
         exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4)
         assert exc_info.value.args[0] == '"lineterminator" must be a string'
diff --git a/pypy/module/_multiprocessing/interp_win32.py b/pypy/module/_multiprocessing/interp_win32.py
--- a/pypy/module/_multiprocessing/interp_win32.py
+++ b/pypy/module/_multiprocessing/interp_win32.py
@@ -109,6 +109,7 @@
         raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
 
 def GetLastError(space):
+    """NOTE: don't use this.  See issue #2658"""
     return space.newint(rwin32.GetLastError_saved())
 
 # __________________________________________________________
diff --git a/pypy/module/_multiprocessing/test/test_connection.py b/pypy/module/_multiprocessing/test/test_connection.py
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ b/pypy/module/_multiprocessing/test/test_connection.py
@@ -12,6 +12,7 @@
                                   'itertools', 'select', 'struct', 'binascii']}
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
     else:
         spaceconfig['usemodules'].append('fcntl')
 
@@ -39,6 +40,10 @@
 
 class BaseConnectionTest(object):
     def test_connection(self):
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         whandle.send_bytes("abc")
@@ -50,6 +55,10 @@
         assert obj == obj2
 
     def test_poll(self):
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         assert rhandle.poll() == False
@@ -64,6 +73,10 @@
 
     def test_read_into(self):
         import array, multiprocessing
+        import sys
+        # if not translated, for win32
+        if not hasattr(sys, 'executable'):
+            sys.executable = 'from test_connection.py'
         rhandle, whandle = self.make_pair()
 
         obj = [1, 2.0, "hello"]
@@ -81,6 +94,7 @@
     }
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
 
     def setup_class(cls):
         if sys.platform != "win32":
@@ -109,6 +123,7 @@
     }
     if sys.platform == 'win32':
         spaceconfig['usemodules'].append('_rawffi')
+        spaceconfig['usemodules'].append('_cffi_backend')
     else:
         spaceconfig['usemodules'].append('fcntl')
 
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize, always_inline
+from rpython.rlib.objectmodel import specialize, always_inline, r_dict
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt
@@ -42,6 +42,22 @@
         ll_res.chars[i] = cast_primitive(UniChar, ch)
     return hlunicode(ll_res)
 
+def slice_eq(a, b):
+    (ll_chars1, start1, length1, _) = a
+    (ll_chars2, start2, length2, _) = b
+    if length1 != length2:
+        return False
+    j = start2
+    for i in range(start1, start1 + length1):
+        if ll_chars1[i] != ll_chars2[j]:
+            return False
+        j += 1
+    return True
+
+def slice_hash(a):
+    (ll_chars, start, length, h) = a
+    return h
+
 TYPE_UNKNOWN = 0
 TYPE_STRING = 1
 class JSONDecoder(object):
@@ -55,7 +71,7 @@
         self.ll_chars = rffi.str2charp(s)
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
-        self.last_type = TYPE_UNKNOWN
+        self.cache = r_dict(slice_eq, slice_hash)
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -248,19 +264,16 @@
 
     def decode_object(self, i):
         start = i
-        w_dict = self.space.newdict()
-        #
+
         i = self.skip_whitespace(i)
         if self.ll_chars[i] == '}':
             self.pos = i+1
-            return w_dict
-        #
+            return self.space.newdict()
+
+        d = {}
         while True:
             # parse a key: value
-            self.last_type = TYPE_UNKNOWN
-            w_name = self.decode_any(i)
-            if self.last_type != TYPE_STRING:
-                self._raise("Key name must be string for object starting at char %d", start)
+            name = self.decode_key(i)
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
@@ -269,13 +282,13 @@
             i = self.skip_whitespace(i)
             #
             w_value = self.decode_any(i)
-            self.space.setitem(w_dict, w_name, w_value)
+            d[name] = w_value
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             i += 1
             if ch == '}':
                 self.pos = i
-                return w_dict
+                return self._create_dict(d)
             elif ch == ',':
                 pass
             elif ch == '\0':
@@ -284,6 +297,9 @@
                 self._raise("Unexpected '%s' when decoding object (char %d)",
                             ch, i-1)
 
+    def _create_dict(self, d):
+        from pypy.objspace.std.dictmultiobject import from_unicode_key_dict
+        return from_unicode_key_dict(self.space, d)
 
     def decode_string(self, i):
         start = i
@@ -295,22 +311,23 @@
             i += 1
             bits |= ord(ch)
             if ch == '"':
-                if bits & 0x80:
-                    # the 8th bit is set, it's an utf8 strnig
-                    content_utf8 = self.getslice(start, i-1)
-                    content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
-                else:
-                    # ascii only, fast path (ascii is a strict subset of
-                    # latin1, and we already checked that all the chars are <
-                    # 128)
-                    content_unicode = strslice2unicode_latin1(self.s, start, i-1)
-                self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.newunicode(content_unicode)
+                return self.space.newunicode(
+                        self._create_string(start, i - 1, bits))
             elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
                 return self.decode_string_escaped(start)
 
+    def _create_string(self, start, end, bits):
+        if bits & 0x80:
+            # the 8th bit is set, it's an utf8 string
+            content_utf8 = self.getslice(start, end)
+            return unicodehelper.decode_utf8(self.space, content_utf8)
+        else:
+            # ascii only, fast path (ascii is a strict subset of
+            # latin1, and we already checked that all the chars are <
+            # 128)
+            return strslice2unicode_latin1(self.s, start, end)
 
     def decode_string_escaped(self, start):
         i = self.pos
@@ -324,7 +341,6 @@
             if ch == '"':
                 content_utf8 = builder.build()
                 content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
-                self.last_type = TYPE_STRING
                 self.pos = i
                 return self.space.newunicode(content_unicode)
             elif ch == '\\':
@@ -387,6 +403,48 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
+    def decode_key(self, i):
+        """ returns an unwrapped unicode """
+        from rpython.rlib.rarithmetic import intmask
+
+        i = self.skip_whitespace(i)
+        ll_chars = self.ll_chars
+        ch = ll_chars[i]
+        if ch != '"':
+            self._raise("Key name must be string at char %d", i)
+        i += 1
+
+        start = i
+        bits = 0
+        strhash = ord(ll_chars[i]) << 7
+        while True:
+            ch = ll_chars[i]
+            i += 1
+            if ch == '"':
+                break
+            elif ch == '\\' or ch < '\x20':
+                self.pos = i-1
+                return self.space.unicode_w(self.decode_string_escaped(start))
+            strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i]))
+            bits |= ord(ch)
+        length = i - start - 1
+        if length == 0:
+            strhash = -1
+        else:
+            strhash ^= length
+            strhash = intmask(strhash)
+        self.pos = i
+        # check cache first:
+        key = (ll_chars, start, length, strhash)
+        try:
+            return self.cache[key]
+        except KeyError:
+            pass
+        res = self._create_string(start, i - 1, bits)
+        self.cache[key] = res
+        return res
+
+
 def loads(space, w_s):
     if space.isinstance_w(w_s, space.w_unicode):
         raise oefmt(space.w_TypeError,
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -5,9 +5,15 @@
 
 import time
 from pypy.interpreter.error import OperationError
-from pypy.module._pypyjson.interp_decoder import loads
+from pypy.module._pypyjson.interp_decoder import loads, JSONDecoder
 from rpython.rlib.objectmodel import specialize, dont_inline
 
+def _create_dict(self, d):
+    w_res = W_Dict()
+    w_res.dictval = d
+    return w_res
+
+JSONDecoder._create_dict = _create_dict
 
 ## MSG = open('msg.json').read()
 
@@ -65,10 +71,14 @@
     def isinstance_w(self, w_x, w_type):
         return isinstance(w_x, w_type)
 
-    def str_w(self, w_x):
+    def bytes_w(self, w_x):
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    def unicode_w(self, w_x):
+        assert isinstance(w_x, W_Unicode)
+        return w_x.unival
+
     @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
@@ -83,13 +93,17 @@
         assert isinstance(key, W_Unicode)
         d.dictval[key.unival] = value
 
-    def wrapunicode(self, x):
+    def newunicode(self, x):
         return W_Unicode(x)
 
-    def wrapint(self, x):
+    def newtext(self, x):
+        return W_String(x)
+    newbytes = newtext
+
+    def newint(self, x):
         return W_Int(x)
 
-    def wrapfloat(self, x):
+    def newfloat(self, x):
         return W_Float(x)
 
     @specialize.argtype(1)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,7 +10,18 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-    
+def test_decode_key():
+    s1 = "123" * 100
+    s = ' "%s"   "%s" ' % (s1, s1)
+    dec = JSONDecoder('fake space', s)
+    assert dec.pos == 0
+    x = dec.decode_key(0)
+    assert x == s1
+    # check caching
+    y = dec.decode_key(dec.pos)
+    assert y == s1
+    assert y is x
+    dec.close()
 
 class AppTest(object):
     spaceconfig = {"objspace.usemodules._pypyjson": True}
@@ -190,6 +201,12 @@
         res = _pypyjson.loads(json)
         assert res == {u'a': u'\ud83d'}
 
+    def test_cache_keys(self):
+        import _pypyjson
+        json = '[{"a": 1}, {"a": 2}]'
+        res = _pypyjson.loads(json)
+        assert res == [{u'a': 1}, {u'a': 2}]
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
@@ -226,7 +243,7 @@
             ('{"spam":[42}', "Unexpected '}' when decoding array (char 11)"),
             ('["]', 'Unterminated string starting at char 1'),
             ('["spam":', "Unexpected ':' when decoding array (char 7)"),
-            ('[{]', "No JSON object could be decoded: unexpected ']' at char 2"),
+            ('[{]', "Key name must be string at char 2"),
         ]
         for inputtext, errmsg in test_cases:
             exc = raises(ValueError, _pypyjson.loads, inputtext)
diff --git a/pypy/module/_socket/test/test_sock_app.py b/pypy/module/_socket/test/test_sock_app.py
--- a/pypy/module/_socket/test/test_sock_app.py
+++ b/pypy/module/_socket/test/test_sock_app.py
@@ -589,7 +589,12 @@
         import _socket
         s = _socket.socket()
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 0
-        assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2) == b'\x00\x00'
+        ret = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 2)
+        if len(ret) == 1:
+            # win32 returns a byte-as-bool
+            assert ret == b'\x00'
+        else:
+            assert ret == b'\x00\x00'
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, True)
         assert s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 0) == 1
         s.setsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1)
@@ -599,7 +604,11 @@
         import _socket
         s = _socket.socket()
         buf = s.getsockopt(_socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1024)
-        assert buf == b'\x00' * 4
+        if len(buf) == 1:
+            # win32 returns a byte-as-bool
+            assert buf == b'\x00'
+        else:
+            assert buf == b'\x00' * 4
         raises(_socket.error, s.getsockopt,
                _socket.IPPROTO_TCP, _socket.TCP_NODELAY, 1025)
         raises(_socket.error, s.getsockopt,
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -996,9 +996,6 @@
         libssl_AUTHORITY_INFO_ACCESS_free(info)
 
 def _get_crl_dp(space, certificate):
-    if OPENSSL_VERSION_NUMBER >= 0x10001000:
-        # Calls x509v3_cache_extensions and sets up crldp
-        libssl_X509_check_ca(certificate)
     dps = rffi.cast(stack_st_DIST_POINT, libssl_X509_get_ext_d2i(
         certificate, NID_crl_distribution_points, None, None))
     if not dps:
@@ -1020,8 +1017,7 @@
                 s_uri = rffi.charpsize2str(uri.c_data, length)
                 cdp_w.append(space.newtext(s_uri))
     finally:
-        if OPENSSL_VERSION_NUMBER < 0x10001000:
-            libssl_sk_DIST_POINT_free(dps)
+        libssl_CRL_DIST_POINTS_free(dps)
     return space.newtuple(cdp_w[:])
 
 def checkwait(space, w_sock, writing):
@@ -1319,7 +1315,7 @@
         if not ctx:
             raise ssl_error(space, "failed to allocate SSL context")
 
-        rgc.add_memory_pressure(10 * 1024 * 1024)
+        rgc.add_memory_pressure(10 * 1024)
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
         self.check_hostname = False
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -1,5 +1,7 @@
 from pypy.interpreter.mixedmodule import MixedModule
 from rpython.rlib.rvmprof import VMProfPlatformUnsupported
+from rpython.translator.platform import CompilationError
+
 
 class Module(MixedModule):
     """
@@ -29,3 +31,9 @@
     import pypy.module._vmprof.interp_vmprof
 except VMProfPlatformUnsupported as e:
     pass
+except CompilationError as e:
+    import sys
+    if sys.platform == 'win32':
+        pass
+    else:
+        raise
diff --git a/pypy/module/_vmprof/conftest.py b/pypy/module/_vmprof/conftest.py
--- a/pypy/module/_vmprof/conftest.py
+++ b/pypy/module/_vmprof/conftest.py
@@ -1,6 +1,8 @@
-import py, platform
+import py, platform, sys
 
 def pytest_collect_directory(path, parent):
     if platform.machine() == 's390x':
-        py.test.skip("zarch tests skipped")
+        py.test.skip("_vmprof tests skipped")
+    if sys.platform == 'win32':
+        py.test.skip("_vmprof tests skipped")
 pytest_collect_file = pytest_collect_directory
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -40,6 +40,7 @@
 from rpython.rlib import rawrefcount
 from rpython.rlib import rthread
 from rpython.rlib.debug import fatalerror_notb
+from rpython.rlib import rstackovf
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 from pypy.module.cpyext.cparser import CTypeSpace
 
@@ -128,6 +129,11 @@
 Py_LT Py_LE Py_EQ Py_NE Py_GT Py_GE Py_TPFLAGS_CHECKTYPES Py_MAX_NDIMS
 PyBUF_FORMAT PyBUF_ND PyBUF_STRIDES PyBUF_WRITABLE
 """.split()

From pypy.commits at gmail.com  Wed Sep 27 11:32:35 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Wed, 27 Sep 2017 08:32:35 -0700 (PDT)
Subject: [pypy-commit] pypy regalloc-playground: adapt to use of constants
 in the new call hint code
Message-ID: <59cbc493.11b51c0a.d4350.e3e1@mx.google.com>

Author: Carl Friedrich Bolz-Tereick <cfbolz at gmx.de>
Branch: regalloc-playground
Changeset: r92478:81d83bb43b83
Date: 2017-09-27 17:31 +0200
http://bitbucket.org/pypy/pypy/changeset/81d83bb43b83/

Log:	adapt to use of constants in the new call hint code

diff --git a/rpython/jit/backend/x86/regalloc.py b/rpython/jit/backend/x86/regalloc.py
--- a/rpython/jit/backend/x86/regalloc.py
+++ b/rpython/jit/backend/x86/regalloc.py
@@ -32,6 +32,15 @@
 from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.jit.backend.x86.regloc import AddressLoc
 
+def compute_gc_level(calldescr, guard_not_forced=False):
+    effectinfo = calldescr.get_extra_info()
+    if guard_not_forced:
+        return SAVE_ALL_REGS
+    elif effectinfo is None or effectinfo.check_can_collect():
+        return SAVE_GCREF_REGS
+    else:
+        return SAVE_DEFAULT_REGS
+
 
 class X86RegisterManager(RegisterManager):
     box_types = [INT, REF]
@@ -846,14 +855,8 @@
             sign_loc = imm1
         else:
             sign_loc = imm0
-        #
-        effectinfo = calldescr.get_extra_info()
-        if guard_not_forced:
-            gc_level = SAVE_ALL_REGS
-        elif effectinfo is None or effectinfo.check_can_collect():
-            gc_level = SAVE_GCREF_REGS
-        else:
-            gc_level = SAVE_DEFAULT_REGS
+
+        gc_level = compute_gc_level(calldescr, guard_not_forced)
         #
         self._call(op, [imm(size), sign_loc] +
                        [self.loc(op.getarg(i)) for i in range(op.numargs())],
diff --git a/rpython/jit/backend/x86/reghint.py b/rpython/jit/backend/x86/reghint.py
--- a/rpython/jit/backend/x86/reghint.py
+++ b/rpython/jit/backend/x86/reghint.py
@@ -10,9 +10,12 @@
     X86_64_SCRATCH_REG, X86_64_XMM_SCRATCH_REG,)
 from rpython.jit.backend.x86 import rx86
 
+from rpython.jit.backend.llsupport.regalloc import (SAVE_DEFAULT_REGS,
+     SAVE_GCREF_REGS, SAVE_ALL_REGS)
 
 from rpython.jit.backend.x86.regalloc import (
-    X86_64_RegisterManager, X86_64_XMMRegisterManager)
+    X86_64_RegisterManager, X86_64_XMMRegisterManager, compute_gc_level
+    )
 
 # tell the register allocator hints about which variables should be placed in
 # what registers (those are just hints, the register allocator will try its
@@ -123,14 +126,7 @@
     def _consider_call(self, op, position, guard_not_forced=False, first_arg_index=1):
         calldescr = op.getdescr()
         assert isinstance(calldescr, CallDescr)
-        effectinfo = calldescr.get_extra_info()
-        # XXX this is nonsense, share the code somehow
-        if guard_not_forced:
-            gc_level = 2
-        elif effectinfo is None or effectinfo.check_can_collect():
-            gc_level = 1
-        else:
-            gc_level = 0
+        gc_level = compute_gc_level(calldescr, guard_not_forced)
         args = op.getarglist()[first_arg_index:]
         argtypes = calldescr.get_arg_types()
         CallHints64(self.longevity).hint(position, args, argtypes, gc_level)
@@ -210,9 +206,10 @@
                         hinted_args.append(arg)
                     next_arg_gpr += 1
         # block all remaining registers that are not caller save
-        # XXX the case save_all_regs == 1 (save callee-save regs + gc ptrs) is
-        # no expressible atm
-        if save_all_regs == 2:
+
+        # XXX the case save_all_regs == SAVE_GCREF_REGS
+        # (save callee-save regs + gc ptrs) is no expressible atm
+        if save_all_regs == SAVE_ALL_REGS:
             regs = X86_64_RegisterManager.all_regs
         else:
             regs = X86_64_RegisterManager.save_around_call_regs

From pypy.commits at gmail.com  Wed Sep 27 13:11:55 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 27 Sep 2017 10:11:55 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: (arigo,
 fijal) start a new branch that tries to account better for tracked
 memory
Message-ID: <59cbdbdb.5ce81c0a.f3219.cb79@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92479:336c50a8852e
Date: 2017-09-27 19:11 +0200
http://bitbucket.org/pypy/pypy/changeset/336c50a8852e/

Log:	(arigo, fijal) start a new branch that tries to account better for
	tracked memory

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1023,7 +1023,7 @@
             if self.max_heap_size < self.next_major_collection_threshold:
                 self.next_major_collection_threshold = self.max_heap_size
 
-    def raw_malloc_memory_pressure(self, sizehint):
+    def raw_malloc_memory_pressure(self, sizehint, adr):
         # Decrement by 'sizehint' plus a very little bit extra.  This
         # is needed e.g. for _rawffi, which may allocate a lot of tiny
         # arrays.
diff --git a/rpython/memory/gc/minimark.py b/rpython/memory/gc/minimark.py
--- a/rpython/memory/gc/minimark.py
+++ b/rpython/memory/gc/minimark.py
@@ -828,7 +828,7 @@
             if self.max_heap_size < self.next_major_collection_threshold:
                 self.next_major_collection_threshold = self.max_heap_size
 
-    def raw_malloc_memory_pressure(self, sizehint):
+    def raw_malloc_memory_pressure(self, sizehint, adr):
         self.next_major_collection_threshold -= sizehint
         if self.next_major_collection_threshold < 0:
             # cannot trigger a full collection now, but we can ensure
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -392,21 +392,22 @@
                 inline = True)
 
         if getattr(GCClass, 'raw_malloc_memory_pressure', False):
-            def raw_malloc_memory_pressure_varsize(length, itemsize):
+            def raw_malloc_memory_pressure_varsize(length, itemsize, adr):
                 totalmem = length * itemsize
                 if totalmem > 0:
-                    gcdata.gc.raw_malloc_memory_pressure(totalmem)
+                    gcdata.gc.raw_malloc_memory_pressure(totalmem, adr)
                 #else: probably an overflow -- the following rawmalloc
                 #      will fail then
-            def raw_malloc_memory_pressure(sizehint):
-                gcdata.gc.raw_malloc_memory_pressure(sizehint)
+            def raw_malloc_memory_pressure(sizehint, adr):
+                gcdata.gc.raw_malloc_memory_pressure(sizehint, adr)
             self.raw_malloc_memory_pressure_varsize_ptr = getfn(
                 raw_malloc_memory_pressure_varsize,
-                [annmodel.SomeInteger(), annmodel.SomeInteger()],
+                [annmodel.SomeInteger(), annmodel.SomeInteger(),
+                 SomeAddress()],
                 annmodel.s_None, minimal_transform = False)
             self.raw_malloc_memory_pressure_ptr = getfn(
                 raw_malloc_memory_pressure,
-                [annmodel.SomeInteger()],
+                [annmodel.SomeInteger(), SomeAddress()],
                 annmodel.s_None, minimal_transform = False)
 
 
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -538,9 +538,14 @@
         if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
             op = hop.spaceop
             size = op.args[0]
+            if len(op.args) == 2:
+                v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
+                    resulttype=llmemory.Address)
+            else:
+                v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
             return hop.genop("direct_call",
-                          [self.raw_malloc_memory_pressure_ptr,
-                           size])
+                              [self.raw_malloc_memory_pressure_ptr,
+                               size, v_adr])
 
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)
@@ -574,9 +579,10 @@
                                                                     c_offset_to_length):
         if flags.get('add_memory_pressure', False):
             if hasattr(self, 'raw_malloc_memory_pressure_varsize_ptr'):
+                v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
                 hop.genop("direct_call",
                           [self.raw_malloc_memory_pressure_varsize_ptr,
-                           v_length, c_item_size])
+                           v_length, c_item_size, v_adr])
         if c_offset_to_length is None:
             if flags.get('zero'):
                 fnptr = self.raw_malloc_varsize_no_length_zero_ptr
diff --git a/rpython/memory/gcwrapper.py b/rpython/memory/gcwrapper.py
--- a/rpython/memory/gcwrapper.py
+++ b/rpython/memory/gcwrapper.py
@@ -83,9 +83,9 @@
     def gettypeid(self, obj):
         return self.get_type_id(lltype.typeOf(obj).TO)
 
-    def add_memory_pressure(self, size):
+    def add_memory_pressure(self, size, adr):
         if hasattr(self.gc, 'raw_malloc_memory_pressure'):
-            self.gc.raw_malloc_memory_pressure(size)
+            self.gc.raw_malloc_memory_pressure(size, adr)
 
     def shrink_array(self, p, smallersize):
         if hasattr(self.gc, 'shrink_array'):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -598,21 +598,26 @@
         return False
     return type(x).__module__ != '__builtin__'   # keep non-builtins
 
-def add_memory_pressure(estimate):
+def add_memory_pressure(estimate, object=None):
     """Add memory pressure for OpaquePtrs."""
     pass
 
 class AddMemoryPressureEntry(ExtRegistryEntry):
     _about_ = add_memory_pressure
 
-    def compute_result_annotation(self, s_nbytes):
+    def compute_result_annotation(self, s_nbytes, s_object=None):
         from rpython.annotator import model as annmodel
         return annmodel.s_None
 
     def specialize_call(self, hop):
-        [v_size] = hop.inputargs(lltype.Signed)
+        v_size = hop.inputarg(lltype.Signed, 0)
+        if len(hop.args_v) == 2:
+            v_obj = hop.inputarg(hop.args_r[1], 1)
+            args = [v_size, v_obj]
+        else:
+            args = [v_size]
         hop.exception_cannot_occur()
-        return hop.genop('gc_add_memory_pressure', [v_size],
+        return hop.genop('gc_add_memory_pressure', args,
                          resulttype=lltype.Void)
 
 
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1613,7 +1613,7 @@
                 digest = ropenssl.EVP_get_digestbyname('sha1')
                 self.ctx = ropenssl.EVP_MD_CTX_new()
                 ropenssl.EVP_DigestInit(self.ctx, digest)
-                rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64)
+                rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + 64, self)
 
             def __del__(self):
                 ropenssl.EVP_MD_CTX_free(self.ctx)

From pypy.commits at gmail.com  Wed Sep 27 15:31:19 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Sep 2017 12:31:19 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into branch
Message-ID: <59cbfc87.c9331c0a.88515.02b3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r92481:24686d8f9ef3
Date: 2017-09-27 21:18 +0300
http://bitbucket.org/pypy/pypy/changeset/24686d8f9ef3/

Log:	merge py3.5 into branch

diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -12,7 +12,8 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
-    CO_GENERATOR, CO_COROUTINE, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
+    CO_GENERATOR, CO_COROUTINE, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY,
+    CO_ITERABLE_COROUTINE)
 from pypy.tool import dis3
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -37,7 +37,8 @@
             CMSG_SPACE CMSG_LEN
             """.split():
 
-            if (name in ('inet_pton', 'inet_ntop', 'socketpair') and
+            if (name in ('inet_pton', 'inet_ntop', 'socketpair',
+                         'CMSG_SPACE', 'CMSG_LEN') and
                 not hasattr(rsocket, name)):
                 continue
 
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,3 +1,4 @@
+import sys
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
 from rpython.rlib.rarithmetic import intmask, r_longlong, r_uint32
@@ -327,41 +328,42 @@
             for (family, socktype, protocol, canonname, addr) in lst]
     return space.newlist(lst1)
 
- at unwrap_spec(size=int)
-def CMSG_SPACE(space, size):
-    """
-    Socket method to determine the optimal byte size of the ancillary.
-    Recommended to be used when computing the ancillary size for recvmsg.
-    :param space:
-    :param size: an integer with the minimum size required.
-    :return: an integer with the minimum memory needed for the required size. The value is memory alligned
-    """
-    if size < 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_SPACE() argument out of range")
-    retval = rsocket.CMSG_SPACE(size)
-    if retval == 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_SPACE() argument out of range")
-    return space.newint(retval)
+if sys.platform != 'win32':
+    @unwrap_spec(size=int)
+    def CMSG_SPACE(space, size):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param space:
+        :param size: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is memory alligned
+        """
+        if size < 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_SPACE() argument out of range")
+        retval = rsocket.CMSG_SPACE(size)
+        if retval == 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_SPACE() argument out of range")
+        return space.newint(retval)
 
- at unwrap_spec(len=int)
-def CMSG_LEN(space, len):
-    """
-    Socket method to determine the optimal byte size of the ancillary.
-    Recommended to be used when computing the ancillary size for recvmsg.
-    :param space:
-    :param len: an integer with the minimum size required.
-    :return: an integer with the minimum memory needed for the required size. The value is not mem alligned.
-    """
-    if len < 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_LEN() argument out of range")
-    retval = rsocket.CMSG_LEN(len)
-    if retval == 0:
-        raise oefmt(space.w_OverflowError,
-                    "CMSG_LEN() argument out of range")
-    return space.newint(retval)
+    @unwrap_spec(len=int)
+    def CMSG_LEN(space, len):
+        """
+        Socket method to determine the optimal byte size of the ancillary.
+        Recommended to be used when computing the ancillary size for recvmsg.
+        :param space:
+        :param len: an integer with the minimum size required.
+        :return: an integer with the minimum memory needed for the required size. The value is not mem alligned.
+        """
+        if len < 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_LEN() argument out of range")
+        retval = rsocket.CMSG_LEN(len)
+        if retval == 0:
+            raise oefmt(space.w_OverflowError,
+                        "CMSG_LEN() argument out of range")
+        return space.newint(retval)
 
 def getdefaulttimeout(space):
     """getdefaulttimeout() -> timeout
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -879,11 +879,14 @@
 socketmethodnames = """
 _accept bind close connect connect_ex fileno detach
 getpeername getsockname getsockopt gettimeout listen
-recv recvfrom recvmsg send sendall sendto sendmsg setblocking
+recv recvfrom send sendall sendto setblocking
 setsockopt settimeout shutdown _reuse _drop recv_into recvfrom_into
 """.split()
 if hasattr(rsocket._c, 'WSAIoctl'):
     socketmethodnames.append('ioctl')
+if rsocket._c.HAVE_SENDMSG:
+    socketmethodnames.append('sendmsg')
+    socketmethodnames.append('recvmsg')
 
 socketmethods = {}
 for methodname in socketmethodnames:
diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py
--- a/pypy/module/cpyext/funcobject.py
+++ b/pypy/module/cpyext/funcobject.py
@@ -18,6 +18,8 @@
     CO_VARKEYWORDS = 0x0008,
     CO_NESTED      = 0x0010,
     CO_GENERATOR   = 0x0020,
+    CO_COROUTINE=0x0080,
+    CO_ITERABLE_COROUTINE=0x0100,
 )
 ALL_CODE_FLAGS = unrolling_iterable(CODE_FLAGS.items())
 
diff --git a/pypy/module/cpyext/genobject.py b/pypy/module/cpyext/genobject.py
--- a/pypy/module/cpyext/genobject.py
+++ b/pypy/module/cpyext/genobject.py
@@ -1,7 +1,32 @@
+from rpython.rtyper.lltypesystem import lltype
 from pypy.interpreter.generator import GeneratorIterator, Coroutine
-from pypy.module.cpyext.api import build_type_checkers
+from pypy.module.cpyext.api import (
+    build_type_checkers, cts, parse_dir, bootstrap_function, slot_function)
+from pypy.module.cpyext.pyobject import PyObject, make_typedescr, as_pyobj
+from pypy.module.cpyext.object import _dealloc
+
+cts.parse_header(parse_dir / 'cpyext_genobject.h')
+
+ at bootstrap_function
+def init_genobject(space):
+    make_typedescr(GeneratorIterator.typedef,
+                   basestruct=cts.gettype('PyGenObject'),
+                   attach=gi_attach,
+                   dealloc=gi_dealloc)
 
 
 PyGen_Check, PyGen_CheckExact = build_type_checkers("Gen", GeneratorIterator)
 
 _, PyCoro_CheckExact = build_type_checkers("Coro", Coroutine)
+
+def gi_attach(space, py_obj, w_obj, w_userdata=None):
+    assert isinstance(w_obj, GeneratorIterator)
+    cts.cast('PyGenObject*', py_obj).c_gi_code = as_pyobj(space, w_obj.pycode)
+
+def gi_realize(space, py_obj):
+    raise NotImplementedError(
+        "PyPy doesn't support creation of generators from the C-API.")
+
+ at slot_function([PyObject], lltype.Void)
+def gi_dealloc(space, py_obj):
+    _dealloc(space, py_obj)
diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -127,6 +127,7 @@
 #include "pycapsule.h"
 #include "bytesobject.h"
 #include "sliceobject.h"
+#include "genobject.h"
 #include "datetime.h"
 #include "pystate.h"
 #include "fileobject.h"
diff --git a/pypy/module/cpyext/include/code.h b/pypy/module/cpyext/include/code.h
--- a/pypy/module/cpyext/include/code.h
+++ b/pypy/module/cpyext/include/code.h
@@ -20,6 +20,11 @@
 #define CO_VARKEYWORDS  0x0008
 #define CO_NESTED       0x0010
 #define CO_GENERATOR    0x0020
+  
+/* The CO_COROUTINE flag is set for coroutine functions (defined with
+   ``async def`` keywords) */
+#define CO_COROUTINE            0x0080
+#define CO_ITERABLE_COROUTINE   0x0100
 
 #define CO_FUTURE_DIVISION         0x02000
 #define CO_FUTURE_ABSOLUTE_IMPORT  0x04000
diff --git a/pypy/module/cpyext/include/genobject.h b/pypy/module/cpyext/include/genobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/genobject.h
@@ -0,0 +1,12 @@
+#ifndef Py_GENOBJECT_H
+#define Py_GENOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "cpyext_genobject.h"
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_GENOBJECT_H */
diff --git a/pypy/module/cpyext/longobject.py b/pypy/module/cpyext/longobject.py
--- a/pypy/module/cpyext/longobject.py
+++ b/pypy/module/cpyext/longobject.py
@@ -201,6 +201,10 @@
     for the conversion.  The radix must be in the range [2, 36]; if it is
     out of range, ValueError will be raised."""
     w_value = space.newunicode(rffi.wcharpsize2unicode(u, length))
+    return PyLong_FromUnicodeObject(space, w_value, base)
+
+ at cpython_api([PyObject, rffi.INT_real], PyObject)
+def PyLong_FromUnicodeObject(space, w_value, base):
     w_base = space.newint(rffi.cast(lltype.Signed, base))
     return space.call_function(space.w_long, w_value, w_base)
 
diff --git a/pypy/module/cpyext/parse/cpyext_genobject.h b/pypy/module/cpyext/parse/cpyext_genobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_genobject.h
@@ -0,0 +1,4 @@
+typedef struct {
+    PyObject_HEAD
+    PyObject* gi_code;
+} PyGenObject;
diff --git a/pypy/module/cpyext/test/test_genobject.py b/pypy/module/cpyext/test/test_genobject.py
--- a/pypy/module/cpyext/test/test_genobject.py
+++ b/pypy/module/cpyext/test/test_genobject.py
@@ -28,7 +28,29 @@
         assert PyCoro_CheckExact(space, w_coroutine)
 
 class AppTestCoroutine(AppTestCpythonExtensionBase):
-    def test_simple(self):
+    def test_generator_coroutine(self):
+        module = self.import_extension('test_gen', [
+            ('is_coroutine', 'METH_O',
+             '''
+             if (!PyGen_CheckExact(args))
+                Py_RETURN_NONE;
+             PyObject* co = ((PyGenObject*)args)->gi_code;
+             if (((PyCodeObject*)co)->co_flags & CO_ITERABLE_COROUTINE)
+                Py_RETURN_TRUE;
+             else
+                Py_RETURN_FALSE;
+             ''')])
+
+        def it():
+            yield 42
+
+        print(module.is_coroutine(it()))
+        assert module.is_coroutine(it()) is False
+        self.debug_collect()  # don't crash while deallocating
+        from types import coroutine
+        assert module.is_coroutine(coroutine(it)()) is True
+
+    def test_await(self):
         """
         module = self.import_extension('test_coroutine', [
             ('await_', 'METH_O',
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -272,6 +272,19 @@
         # A string with arabic digits. 'BAD' is after the 6th character.
         assert module.from_unicode(u'  1\u0662\u0663\u0664BAD') == (1234, 4660)
 
+    def test_fromunicodeobject(self):
+        module = self.import_extension('foo', [
+            ("from_unicodeobject", "METH_O",
+             """
+                 return Py_BuildValue("NN",
+                     PyLong_FromUnicodeObject(args, 10),
+                     PyLong_FromUnicodeObject(args, 16));
+             """),
+            ])
+        # A string with arabic digits.
+        assert (module.from_unicodeobject(u'  1\u0662\u0663\u0664')
+                == (1234, 4660))
+
     def test_aslong(self):
         module = self.import_extension('foo', [
             ("as_long", "METH_O",
diff --git a/pypy/module/cpyext/test/test_unicodeobject.py b/pypy/module/cpyext/test/test_unicodeobject.py
--- a/pypy/module/cpyext/test/test_unicodeobject.py
+++ b/pypy/module/cpyext/test/test_unicodeobject.py
@@ -851,6 +851,17 @@
         assert r"['a\n', 'b\n', 'c\n', 'd']" == space.unwrap(space.repr(
                 PyUnicode_Splitlines(space, w_str, 1)))
 
+    def test_substring_api(self, space):
+        w_str = space.wrap(u"abcd")
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 1, 3)) == u"bc"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 0, 4)) == u"abcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 0, 9)) == u"abcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 1, 4)) == u"bcd"
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 2, 2)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 5, 4)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 5, 3)) == u""
+        assert space.unwrap(PyUnicode_Substring(space, w_str, 4, 3)) == u""
+
     def test_Ready(self, space):
         w_str = space.wrap(u'abc')  # ASCII
         py_str = as_pyobj(space, w_str)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -1043,3 +1043,17 @@
     resulting strings."""
     w_keepend = space.newbool(bool(rffi.cast(lltype.Signed, keepend)))
     return space.call_method(w_str, "splitlines", w_keepend)
+
+ at cpython_api([PyObject, Py_ssize_t, Py_ssize_t], PyObject)
+def PyUnicode_Substring(space, w_str, start, end):
+    usrc = space.unicode_w(w_str)
+    length = len(usrc)
+    if start < 0 or end < 0:
+        raise oefmt(space.w_IndexError, "string index out of range")
+    if start >= length or end < start:
+        result = u''
+    else:
+        if end > length:
+            end = length
+        result = usrc[start:end]
+    return space.newunicode(result)
diff --git a/rpython/rlib/_rsocket_rffi.py b/rpython/rlib/_rsocket_rffi.py
--- a/rpython/rlib/_rsocket_rffi.py
+++ b/rpython/rlib/_rsocket_rffi.py
@@ -348,7 +348,8 @@
                                  ('ifr_name', rffi.CFixedArray(rffi.CHAR, 8))])
 
 # insert handler for sendmsg / recvmsg here
-if _POSIX:
+HAVE_SENDMSG = bool(_POSIX)
+if HAVE_SENDMSG:
     includes = ['stddef.h',
                 'sys/socket.h',
                 'unistd.h',
diff --git a/rpython/rlib/rsocket.py b/rpython/rlib/rsocket.py
--- a/rpython/rlib/rsocket.py
+++ b/rpython/rlib/rsocket.py
@@ -1393,7 +1393,7 @@
         return (make_socket(fd0, family, type, proto, SocketClass),
                 make_socket(fd1, family, type, proto, SocketClass))
 
-if _c._POSIX:
+if _c.HAVE_SENDMSG:
     def CMSG_LEN( demanded_len):
         """
         Socket method to determine the optimal byte size of the ancillary.

From pypy.commits at gmail.com  Wed Sep 27 15:31:21 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Sep 2017 12:31:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: update release notes for latest merges
Message-ID: <59cbfc89.ccb2df0a.d412f.2ebb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92482:46f3190ace68
Date: 2017-09-27 21:27 +0300
http://bitbucket.org/pypy/pypy/changeset/46f3190ace68/

Log:	update release notes for latest merges

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -24,7 +24,7 @@
 memory use to 50% and increase parsing speed by up to 15% for large JSON files
 with many repeating dictionary keys (which is quite common).
 
-CFFI_, which is part of the PyPy release, has been updated to 1.11,
+CFFI_, which is part of the PyPy release, has been updated to 1.11.1,
 improving an already great package for interfacing with C. CFFI now supports
 complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has
 improved support for callbacks.
@@ -166,7 +166,8 @@
 
   * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
     ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
-    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``
+    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``,
+    ``PyGenObject``, ``PyUnicode_Substring``, ``PyLong_FromUnicodeObject``
   * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
   * Support the new version of ``os.stat()`` on win32
   * Use ``stat3()`` on Posix

From pypy.commits at gmail.com  Wed Sep 27 15:31:23 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Sep 2017 12:31:23 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release
 for cffi 1.11.1
Message-ID: <59cbfc8b.e6361c0a.8267a.929d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r92483:fedadd23d48b
Date: 2017-09-27 22:24 +0300
http://bitbucket.org/pypy/pypy/changeset/fedadd23d48b/

Log:	merge default into release for cffi 1.11.1

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -214,7 +214,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -27,8 +27,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -69,6 +69,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -77,14 +78,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -123,6 +123,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -149,11 +150,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -161,6 +164,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -171,6 +175,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -181,14 +186,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -210,7 +213,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -242,6 +244,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -272,6 +275,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -292,12 +296,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.9.0.rst
    release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.9.0.rst
    whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
@@ -36,6 +37,7 @@
 .. toctree::
 
    whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
 
diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.9.0.rst
@@ -0,0 +1,213 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.9 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 includes the upstream stdlib version 3.5.3.
+
+NumPy and Pandas now work on PyPy2.7. Issues that appeared as excessive memory
+use were cleared up and other incompatibilities were resolved. The C-API
+compatibility layer does slow down code which crosses the python-c interface
+often, we have ideas on how it could be improved, and still recommend
+using pure python on PyPy or interfacing via CFFI_. Many other modules
+based on C-API exentions now work on PyPy as well.
+
+Cython 0.27 (released last week) should support more projects with PyPy, both
+on PyPy2.7 and PyPy3.5 beta.
+
+We optimized the JSON parser for recurring string keys, which should decrease
+memory use to 50% and increase parsing speed by up to 15% for large JSON files
+with many repeating dictionary keys (which is quite common).
+
+CFFI_, which is part of the PyPy release, has been updated to 1.11.1,
+improving an already great package for interfacing with C. CFFI now supports
+complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has
+improved support for callbacks.
+
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy. Of course the bug fixes and performance enhancements
+mentioned above are part of both PyPy2.7 and PyPy3.5 beta.
+
+As always, this release fixed many other issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.9 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _vmprof: http://vmprof.readthedocs.io
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017)
+======================================================================================
+
+See also issues that were resolved_
+
+Note that these are also merged into PyPy 3.5
+
+* New features and cleanups
+
+  * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``,
+    ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``,
+    ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat``
+  * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible
+  * ``gcdump`` now reports largest object
+  * More complete support in the ``_curses`` CFFI module
+  * Add cPickle.Unpickler.find_global (issue 1853_)
+  * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set
+  * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()``
+  * Update RevDB to version pypy2.7-v5.6.2
+  * Previously, ``instance.method`` would return always the same bound method
+    object, when gotten from the same instance (as far as ``is`` and ``id()``
+    can tell).  CPython doesn't do that.  Now PyPy, like CPython, returns a 
+    different bound method object every time.  For ``type.method``, PyPy2 still
+    returns always the same *unbound* method object; CPython does it for built-in
+    types but not for user-defined types
+  * Link to disable PaX protection for the JIT when needed
+  * Update build instructions and an rarely used Makefile
+  * Recreate support for using leakfinder in cpyext tests which had suffered
+    bit-rot, disable due to many false positives
+  * Add more functionality to ``sysconfig``
+  * Added ``_swappedbytes_`` support for ``ctypes.Structure``
+  * Better support the ``inspect`` module on ``frames``
+
+* Bug Fixes 
+
+  * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
+  * Implement ``PyListOjbect.getstorage_copy``
+  * Fix for ``reversed(dictproxy)`` issue 2601_
+  * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_
+  * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
+  * ``gc.get_objects`` now handles objects with finalizers more consistently
+  * Fixed memory leak in ``SSLContext.getpeercert`` returning validated
+    certificates and ``SSLContext.get_ca_certs(binary_mode=True)``
+    (_get_crl_dp) `CPython issue 29738`_
+
+* Performance improvements:
+
+  * Improve performance of ``bytearray.extend`` by rewriting portions in app-level
+  * Optimize list accesses with constant indexes better by retaining more
+    information about them
+  * Add a jit driver for ``array.count`` and ``array.index``
+  * Improve information retained in a bridge wrt ``array``
+  * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into
+    pure C macros
+  * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
+    all the ints
+  * Cache string keys that occur in JSON dicts, as they are likely to repeat
+
+* RPython improvements
+
+  * Do not preallocate a RPython list if we only know an upper bound on its size
+  * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
+  * Replace magical NOT RPYTHON comment with a decorator
+  * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
+
+* Degredations
+
+  * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module
+
+.. _here: cpython_differences.html
+.. _1853: https://bitbucket.org/pypy/pypy/issues/1853
+.. _2592: https://bitbucket.org/pypy/pypy/issues/2592
+.. _2590: https://bitbucket.org/pypy/pypy/issues/2590
+.. _2621: https://bitbucket.org/pypy/pypy/issues/2621
+
+Highlights of the PyPy3.5 release (since 5.8 beta released June 2017)
+======================================================================
+
+* New features
+
+  * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
+    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
+    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``,
+    ``PyGenObject``, ``PyUnicode_Substring``, ``PyLong_FromUnicodeObject``
+  * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
+  * Support the new version of ``os.stat()`` on win32
+  * Use ``stat3()`` on Posix
+  * Accept buffer objects as filenames, except for `oslistdir``
+  * Make slices of array ``memoryview`` s usable as writable buffers if contiguous
+  * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded
+  * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version
+  * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence
+  * Enable PGO for CLang
+  * Rework ``cppyy`` packaging and rename the backend to ``_cppyy``
+  * Support for libressl 2.5.4
+  * Mirror CPython ``classmethod __reduce__`` which fixes pickling test
+  * Use utf-8 for ``readline`` history file
+  * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses
+  * Add async slot functions in cpyext
+
+* Bug Fixes
+
+  * Try to make ``openssl`` CFFI bindings more general and future-proof
+  * Better support ``importlib`` by only listing built-in modules in ``sys.builtin``
+  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_
+  * Fix for ``reversed(mapping object)`` issue 2601_
+  * Fixing regression with non-started generator receiving non-``None``, should
+    always raise ``TypeError``
+  * ``itertools.islice``: use same logic as CPython, fixes 2643_
+
+* Performance improvements:
+
+  * 
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+
+.. _resolved: whatsnew-pypy2-5.9.0.html
+.. _2579: https://bitbucket.org/pypy/pypy/issues/2579
+.. _2601: https://bitbucket.org/pypy/pypy/issues/2601
+.. _2643: https://bitbucket.org/pypy/pypy/issues/2643
+.. _CPython issue 29738: https://bugs.python.org/issue29738
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst
copy from pypy/doc/whatsnew-head.rst
copy to pypy/doc/whatsnew-pypy2-5.9.0.rst
diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst
@@ -0,0 +1,7 @@
+=======================
+What's new in PyPy3 5.9
+=======================
+
+.. this is the revision after release-pypy3.5-5.8
+.. startrev: afbf09453369
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,7 +1,7 @@
 =========================
-What's new in PyPy3 5.7+
+What's new in PyPy3 5.9+
 =========================
 
-.. this is the revision after release-pypy3.3-5.7.x was branched
-.. startrev: afbf09453369
+.. this is the revision after release-pypy3.5-5.9
+.. startrev: be41e3ac0a29
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.11.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1315,7 +1315,7 @@
         if not ctx:
             raise ssl_error(space, "failed to allocate SSL context")
 
-        rgc.add_memory_pressure(10 * 1024 * 1024)
+        rgc.add_memory_pressure(10 * 1024)
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
         self.check_hostname = False
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1314,6 +1314,18 @@
     decls = defaultdict(list)
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
+    decls[pypy_decl].append("""
+        /* hack for https://bugs.python.org/issue29943 */
+        PyAPI_FUNC(int) %s(PySliceObject *arg0,
+                           Signed arg1, Signed *arg2,
+                           Signed *arg3, Signed *arg4, Signed *arg5);
+        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+                Py_ssize_t *arg5) {
+            return %s(arg0, arg1, arg2, arg3,
+                      arg4, arg5);
+        }
+    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]

From pypy.commits at gmail.com  Wed Sep 27 15:31:25 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Sep 2017 12:31:25 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix for c-89 (visual 9) compilation
Message-ID: <59cbfc8d.499edf0a.8ed97.59df@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92484:611fac54ebfd
Date: 2017-09-23 23:51 +0300
http://bitbucket.org/pypy/pypy/changeset/611fac54ebfd/

Log:	fix for c-89 (visual 9) compilation

diff --git a/pypy/module/cpyext/test/test_memoryobject.py b/pypy/module/cpyext/test/test_memoryobject.py
--- a/pypy/module/cpyext/test/test_memoryobject.py
+++ b/pypy/module/cpyext/test/test_memoryobject.py
@@ -72,7 +72,7 @@
              """
              /* Create an approximation of the buffer for a 0d ndarray */
              Py_buffer buf;
-             PyObject *str = PyBytes_FromString("hello, world.");
+             PyObject *ret, *str = PyBytes_FromString("hello, world.");
              buf.buf = PyBytes_AsString(str);
              buf.obj = str;
              buf.readonly = 1;
@@ -80,7 +80,7 @@
              buf.itemsize = 13;
              buf.ndim = 0;
              buf.shape = NULL;
-             PyObject* ret = PyMemoryView_FromBuffer(&buf);
+             ret = PyMemoryView_FromBuffer(&buf);
              return ret;
             """)])
         result = module.create_view()

From pypy.commits at gmail.com  Wed Sep 27 15:31:16 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 27 Sep 2017 12:31:16 -0700 (PDT)
Subject: [pypy-commit] pypy default: be more positive about numpy,pandas
Message-ID: <59cbfc84.0988df0a.e9c9e.43cd@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92480:c1b14bf66a8e
Date: 2017-09-27 21:12 +0300
http://bitbucket.org/pypy/pypy/changeset/c1b14bf66a8e/

Log:	be more positive about numpy,pandas

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -10,9 +10,12 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-Only a handful of failing tests remain in NumPy and Pandas on PyPy2.7, issues
-that appeared as excessive memory use were cleared up and other incompatibilities
-were resolved.
+NumPy and Pandas now work on PyPy2.7. Issues that appeared as excessive memory
+use were cleared up and other incompatibilities were resolved. The C-API
+compatibility layer does slow down code which crosses the python-c interface
+often, we have ideas on how it could be improved, and still recommend
+using pure python on PyPy or interfacing via CFFI_. Many other modules
+based on C-API exentions now work on PyPy as well.
 
 Cython 0.27 (released last week) should support more projects with PyPy, both
 on PyPy2.7 and PyPy3.5 beta.

From pypy.commits at gmail.com  Wed Sep 27 15:43:07 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Wed, 27 Sep 2017 12:43:07 -0700 (PDT)
Subject: [pypy-commit] pypy default: unscientific memory pressure estimate
 of =~ 6k for SSL*
Message-ID: <59cbff4b.d5c21c0a.875be.0381@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r92485:4302fe3d912e
Date: 2017-09-27 12:41 -0700
http://bitbucket.org/pypy/pypy/changeset/4302fe3d912e/

Log:	unscientific memory pressure estimate of =~ 6k for SSL*

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -283,6 +283,7 @@
         sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
         self.ssl = libssl_SSL_new(w_ctx.ctx)  # new ssl struct
 
+        rgc.add_memory_pressure(6 * 1024)
         self.register_finalizer(space)
 
         index = compute_unique_id(self)

From pypy.commits at gmail.com  Wed Sep 27 15:43:09 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Wed, 27 Sep 2017 12:43:09 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default
Message-ID: <59cbff4d.4db6df0a.c58c6.2a54@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: release-pypy2.7-5.x
Changeset: r92486:e3ab7b0bd8ac
Date: 2017-09-27 12:42 -0700
http://bitbucket.org/pypy/pypy/changeset/e3ab7b0bd8ac/

Log:	merge default

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -283,6 +283,7 @@
         sock_fd = space.int_w(space.call_method(w_sock, "fileno"))
         self.ssl = libssl_SSL_new(w_ctx.ctx)  # new ssl struct
 
+        rgc.add_memory_pressure(6 * 1024)
         self.register_finalizer(space)
 
         index = compute_unique_id(self)

From pypy.commits at gmail.com  Wed Sep 27 18:38:11 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Wed, 27 Sep 2017 15:38:11 -0700 (PDT)
Subject: [pypy-commit] pypy default: blurb 4302fe3d912e & b67bc051d0c1
Message-ID: <59cc2853.2483df0a.34088.04fb@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r92487:943002f74290
Date: 2017-09-27 15:37 -0700
http://bitbucket.org/pypy/pypy/changeset/943002f74290/

Log:	blurb 4302fe3d912e & b67bc051d0c1

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -148,6 +148,8 @@
   * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
   * Replace magical NOT RPYTHON comment with a decorator
   * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
+  * Reduce excessive ``memory_pressure`` for ``_SSLContext`` objects and add
+    ``memory_pressure`` for ``_SSLSocket`` objects
 
 * Degredations
 

From pypy.commits at gmail.com  Thu Sep 28 01:58:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 27 Sep 2017 22:58:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2669
Message-ID: <59cc8f90.f68adf0a.1c070.50e3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92488:7ee8c3a718e5
Date: 2017-09-28 07:57 +0200
http://bitbucket.org/pypy/pypy/changeset/7ee8c3a718e5/

Log:	Issue #2669

	Make Py_FileSystemDefaultEncoding non-NULL in cpyext.

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -558,6 +558,7 @@
     'PyObject_GetBuffer', 'PyBuffer_Release',
     'PyBuffer_FromMemory', 'PyBuffer_FromReadWriteMemory', 'PyBuffer_FromObject',
     'PyBuffer_FromReadWriteObject', 'PyBuffer_New', 'PyBuffer_Type', '_Py_get_buffer_type',
+    '_Py_setfilesystemdefaultencoding',
 
     'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr',
     'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr',
@@ -1045,11 +1046,17 @@
     get_capsule_type = rffi.llexternal('_%s_get_capsule_type' % prefix,
                                        [], PyTypeObjectPtr,
                                        compilation_info=eci, _nowrapper=True)
+    setdefenc = rffi.llexternal('_%s_setfilesystemdefaultencoding' % prefix,
+                                [rffi.CCHARP], lltype.Void,
+                                compilation_info=eci, _nowrapper=True)
     def init_types(space):
         from pypy.module.cpyext.typeobject import py_type_ready
+        from pypy.module.sys.interp_encoding import getfilesystemencoding
         py_type_ready(space, get_buffer_type())
         py_type_ready(space, get_cobject_type())
         py_type_ready(space, get_capsule_type())
+        s = space.text_w(getfilesystemencoding(space))
+        setdefenc(rffi.str2charp(s, track_allocation=False))  # "leaks"
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
     _reinit_tls = rffi.llexternal('%sThread_ReInitTLS' % prefix, [],
diff --git a/pypy/module/cpyext/include/fileobject.h b/pypy/module/cpyext/include/fileobject.h
--- a/pypy/module/cpyext/include/fileobject.h
+++ b/pypy/module/cpyext/include/fileobject.h
@@ -1,1 +1,2 @@
-#define Py_FileSystemDefaultEncoding   NULL
+PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding;
+PyAPI_FUNC(void) _Py_setfilesystemdefaultencoding(const char *);
diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c
--- a/pypy/module/cpyext/src/missing.c
+++ b/pypy/module/cpyext/src/missing.c
@@ -27,3 +27,7 @@
 int Py_Py3kWarningFlag = 0;
 int Py_HashRandomizationFlag = 0;
 
+const char *Py_FileSystemDefaultEncoding;  /* filled when cpyext is imported */
+void _Py_setfilesystemdefaultencoding(const char *enc) {
+    Py_FileSystemDefaultEncoding = enc;
+}
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test/test_fileobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_fileobject.py
@@ -0,0 +1,13 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestFileObject(AppTestCpythonExtensionBase):
+    def test_defaultencoding(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("defenc", "METH_NOARGS",
+             """
+                return PyString_FromString(Py_FileSystemDefaultEncoding);
+             """),
+            ])
+        assert module.defenc() == sys.getfilesystemencoding()

From pypy.commits at gmail.com  Thu Sep 28 11:55:30 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 28 Sep 2017 08:55:30 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: write the number of raw
 malloced bytes into structs that are used for add_memory_pressure
Message-ID: <59cd1b72.51421c0a.5f9f4.aed5@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92489:238b67adb061
Date: 2017-09-28 17:54 +0200
http://bitbucket.org/pypy/pypy/changeset/238b67adb061/

Log:	write the number of raw malloced bytes into structs that are used
	for add_memory_pressure

diff --git a/rpython/annotator/bookkeeper.py b/rpython/annotator/bookkeeper.py
--- a/rpython/annotator/bookkeeper.py
+++ b/rpython/annotator/bookkeeper.py
@@ -71,6 +71,7 @@
 
         self.needs_generic_instantiate = {}
         self.thread_local_fields = set()
+        self.memory_pressure_types = set()
 
         self.register_builtins()
 
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -832,6 +832,22 @@
 
     gct_fv_gc_malloc_varsize = gct_fv_gc_malloc
 
+    def gct_gc_add_memory_pressure(self, hop):
+        if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
+            op = hop.spaceop
+            size = op.args[0]
+            if len(op.args) == 2:
+                v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure")
+                hop.genop("bare_setfield", [op.args[1], v_fld, size])
+                v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
+                    resulttype=llmemory.Address)
+            else:
+                v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
+            return hop.genop("direct_call",
+                              [self.raw_malloc_memory_pressure_ptr,
+                               size, v_adr])
+
+
     def gct_gc__collect(self, hop):
         op = hop.spaceop
         if len(op.args) == 1:
diff --git a/rpython/memory/gctransform/transform.py b/rpython/memory/gctransform/transform.py
--- a/rpython/memory/gctransform/transform.py
+++ b/rpython/memory/gctransform/transform.py
@@ -535,17 +535,7 @@
         return self.varsize_malloc_helper(hop, flags, meth, [])
 
     def gct_gc_add_memory_pressure(self, hop):
-        if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
-            op = hop.spaceop
-            size = op.args[0]
-            if len(op.args) == 2:
-                v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
-                    resulttype=llmemory.Address)
-            else:
-                v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
-            return hop.genop("direct_call",
-                              [self.raw_malloc_memory_pressure_ptr,
-                               size, v_adr])
+        pass
 
     def varsize_malloc_helper(self, hop, flags, meth, extraargs):
         def intconst(c): return rmodel.inputconst(lltype.Signed, c)
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -607,6 +607,11 @@
 
     def compute_result_annotation(self, s_nbytes, s_object=None):
         from rpython.annotator import model as annmodel
+        if s_object is not None:
+            if not isinstance(s_object, annmodel.SomeInstance):
+                raise Exception("Wrong kind of object passed to "
+                                "add memory pressure")
+            self.bookkeeper.memory_pressure_types.add(s_object.classdef)
         return annmodel.s_None
 
     def specialize_call(self, hop):
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -523,6 +523,10 @@
                 if not attrdef.readonly and self.is_quasi_immutable(name):
                     llfields.append(('mutate_' + name, OBJECTPTR))
 
+            bookkeeper = self.rtyper.annotator.bookkeeper
+            if self.classdef in bookkeeper.memory_pressure_types:
+                llfields = [('special_memory_pressure', lltype.Signed)] + llfields
+
             object_type = MkStruct(self.classdef.name,
                                    ('super', self.rbase.object_type),
                                    hints=hints,

From pypy.commits at gmail.com  Thu Sep 28 12:57:46 2017
From: pypy.commits at gmail.com (fijal)
Date: Thu, 28 Sep 2017 09:57:46 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: whack whack whack until we
 get to the point of actually calling GC stuff
Message-ID: <59cd2a0a.b399df0a.d1e92.4323@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92490:8e15f0778f6e
Date: 2017-09-28 18:57 +0200
http://bitbucket.org/pypy/pypy/changeset/8e15f0778f6e/

Log:	whack whack whack until we get to the point of actually calling GC
	stuff

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -72,6 +72,7 @@
 from rpython.rlib.rarithmetic import LONG_BIT_SHIFT
 from rpython.rlib.debug import ll_assert, debug_print, debug_start, debug_stop
 from rpython.rlib.objectmodel import specialize
+from rpython.rlib import rgc
 from rpython.memory.gc.minimarkpage import out_of_memory
 
 #
@@ -2918,6 +2919,9 @@
         self.old_objects_with_weakrefs.delete()
         self.old_objects_with_weakrefs = new_with_weakref
 
+    def get_stats(self, stats_no):
+        return 0
+
 
     # ----------
     # RawRefCount
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -410,6 +410,12 @@
                 [annmodel.SomeInteger(), SomeAddress()],
                 annmodel.s_None, minimal_transform = False)
 
+        if getattr(GCClass, 'get_stats', False):
+            def get_stats(stats_no):
+                return gcdata.gc.get_stats(stats_no)
+            self.get_stats_ptr = getfn(get_stats, [annmodel.SomeInteger()],
+                annmodel.SomeInteger())
+
 
         self.identityhash_ptr = getfn(GCClass.identityhash.im_func,
                                       [s_gc, s_gcref],
@@ -843,9 +849,17 @@
                     resulttype=llmemory.Address)
             else:
                 v_adr = rmodel.inputconst(llmemory.Address, llmemory.NULL)
+            hop.genop("direct_call", [self.raw_malloc_memory_pressure_ptr,
+                               size, v_adr])
+
+
+    def gct_gc_get_stats(self, hop):
+        if hasattr(self, 'get_stats_ptr'):
             return hop.genop("direct_call",
-                              [self.raw_malloc_memory_pressure_ptr,
-                               size, v_adr])
+                [self.get_stats_ptr, hop.spaceop.args[0]],
+                resultvar=hop.spaceop.result)
+        hop.genop("same_as", [rmodel.inputconst(lltype.Signed, 0)],
+            resultvar=hop.spaceop.result)
 
 
     def gct_gc__collect(self, hop):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -650,6 +650,12 @@
     else:
         return id(gcref._x)
 
+TOTAL_MEMORY, = range(1)
+
+ at not_rpython
+def get_stats(stat_no):
+    raise NotImplementedError
+
 @not_rpython
 def dump_rpy_heap(fd):
     raise NotImplementedError
@@ -844,6 +850,18 @@
         return hop.genop('gc_get_rpy_type_index', vlist,
                          resulttype = hop.r_result)
 
+class Entry(ExtRegistryEntry):
+    _about_ = get_stats
+    def compute_result_annotation(self, s_no):
+        from rpython.annotator.model import SomeInteger
+        if not isinstance(s_no, SomeInteger):
+            raise Exception("expecting an integer")
+        return SomeInteger()
+    def specialize_call(self, hop):
+        args = hop.inputargs(lltype.Signed)
+        hop.exception_cannot_occur()
+        return hop.genop('gc_get_stats', args, resulttype=lltype.Signed)
+
 @not_rpython
 def _is_rpy_instance(gcref):
     raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -485,6 +485,7 @@
     'gc_gettypeid'        : LLOp(),
     'gc_gcflag_extra'     : LLOp(),
     'gc_add_memory_pressure': LLOp(),
+    'gc_get_stats'        : LLOp(),
     'gc_fq_next_dead'     : LLOp(),
     'gc_fq_register'      : LLOp(),
     'gc_ignore_finalizer' : LLOp(canrun=True),
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1625,7 +1625,7 @@
                 am2 = am1
                 am1 = A()
             # what can we use for the res?
-            return 0
+            return rgc.get_stats(rgc.TOTAL_MEMORY)
         return f
 
     def test_nongc_opaque_attached_to_gc(self):

From pypy.commits at gmail.com  Thu Sep 28 15:49:19 2017
From: pypy.commits at gmail.com (mjacob)
Date: Thu, 28 Sep 2017 12:49:19 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59cd523f.6594df0a.e0a49.67cd@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r92491:aee2981816b8
Date: 2017-09-28 21:48 +0200
http://bitbucket.org/pypy/pypy/changeset/aee2981816b8/

Log:	hg merge default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except (AttributeError, StopIteration):
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -247,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -215,7 +215,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -27,8 +27,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -69,6 +69,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -77,14 +78,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -123,6 +123,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -149,11 +150,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -161,6 +164,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -171,6 +175,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -181,14 +186,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -210,7 +213,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -242,6 +244,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -272,6 +275,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -292,12 +296,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -436,7 +436,8 @@
 
 * the ``__builtins__`` name is always referencing the ``__builtin__`` module,
   never a dictionary as it sometimes is in CPython. Assigning to
-  ``__builtins__`` has no effect.
+  ``__builtins__`` has no effect.  (For usages of tools like
+  RestrictedPython, see `issue #2653`_.)
 
 * directly calling the internal magic methods of a few built-in types
   with invalid arguments may have a slightly different result.  For
@@ -556,4 +557,4 @@
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
-
+.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/
diff --git a/pypy/doc/index-of-release-notes.rst b/pypy/doc/index-of-release-notes.rst
--- a/pypy/doc/index-of-release-notes.rst
+++ b/pypy/doc/index-of-release-notes.rst
@@ -6,6 +6,7 @@
 
 .. toctree::
 
+   release-v5.9.0.rst
    release-v5.8.0.rst
    release-v5.7.1.rst
    release-v5.7.0.rst
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -7,6 +7,7 @@
 .. toctree::
 
    whatsnew-head.rst
+   whatsnew-pypy2-5.9.0.rst
    whatsnew-pypy2-5.8.0.rst
    whatsnew-pypy2-5.7.0.rst
    whatsnew-pypy2-5.6.0.rst
@@ -36,6 +37,7 @@
 .. toctree::
 
    whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.9.0.rst
    whatsnew-pypy3-5.8.0.rst
    whatsnew-pypy3-5.7.0.rst
 
diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/release-v5.9.0.rst
@@ -0,0 +1,215 @@
+=====================================
+PyPy2.7 and PyPy3.5 v5.9 dual release
+=====================================
+
+The PyPy team is proud to release both PyPy2.7 v5.9 (an interpreter supporting
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.9 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
+the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
+
+This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
+PyPy3.5 includes the upstream stdlib version 3.5.3.
+
+NumPy and Pandas now work on PyPy2.7. Issues that appeared as excessive memory
+use were cleared up and other incompatibilities were resolved. The C-API
+compatibility layer does slow down code which crosses the python-c interface
+often, we have ideas on how it could be improved, and still recommend
+using pure python on PyPy or interfacing via CFFI_. Many other modules
+based on C-API exentions now work on PyPy as well.
+
+Cython 0.27 (released last week) should support more projects with PyPy, both
+on PyPy2.7 and PyPy3.5 beta.
+
+We optimized the JSON parser for recurring string keys, which should decrease
+memory use to 50% and increase parsing speed by up to 15% for large JSON files
+with many repeating dictionary keys (which is quite common).
+
+CFFI_, which is part of the PyPy release, has been updated to 1.11.1,
+improving an already great package for interfacing with C. CFFI now supports
+complex arguments in API mode, as well as ``char16_t`` and ``char32_t`` and has
+improved support for callbacks.
+
+Please let us know if your use case is slow, we have ideas how to make things
+faster but need real-world examples (not micro-benchmarks) of problematic code.
+
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy. Of course the bug fixes and performance enhancements
+mentioned above are part of both PyPy2.7 and PyPy3.5 beta.
+
+As always, this release fixed many other issues and bugs raised by the
+growing community of PyPy users. We strongly recommend updating.
+
+You can download the v5.9 releases here:
+
+    http://pypy.org/download.html
+
+We would like to thank our donors for the continued support of the PyPy
+project.
+
+We would also like to thank our contributors and
+encourage new people to join the project. PyPy has many
+layers and we need help with all of them: `PyPy`_ and `RPython`_ documentation
+improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
+with making RPython's JIT even better.
+
+.. _vmprof: http://vmprof.readthedocs.io
+.. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
+.. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
+.. _`PyPy`: index.html
+.. _`RPython`: https://rpython.readthedocs.org
+.. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
+.. _`help`: project-ideas.html
+
+What is PyPy?
+=============
+
+PyPy is a very compliant Python interpreter, almost a drop-in replacement for
+CPython 2.7 and CPython 3.5. It's fast (`PyPy and CPython 2.7.x`_ performance comparison)
+due to its integrated tracing JIT compiler.
+
+We also welcome developers of other `dynamic languages`_ to see what RPython
+can do for them.
+
+The PyPy 2.7 release supports: 
+
+  * **x86** machines on most common operating systems
+    (Linux 32/64 bits, Mac OS X 64 bits, Windows 32 bits, OpenBSD, FreeBSD)
+  
+  * newer **ARM** hardware (ARMv6 or ARMv7, with VFPv3) running Linux,
+  
+  * big- and little-endian variants of **PPC64** running Linux,
+
+  * **s390x** running Linux
+
+.. _`PyPy and CPython 2.7.x`: http://speed.pypy.org
+.. _`dynamic languages`: http://rpython.readthedocs.io/en/latest/examples.html
+
+Highlights of the PyPy2.7, cpyext, and RPython changes (since 5.8 released June, 2017)
+======================================================================================
+
+See also issues that were resolved_
+
+Note that these are also merged into PyPy 3.5
+
+* New features and cleanups
+
+  * Add support for ``PyFrozenSet_New``, ``PyObject_HashNotImplemented``,
+    ``PyObject_Print(NULL, ...)``, ``PyObject_RichCompareBool(a, a, ...)``,
+    ``PyType_IS_GC`` (does nothing), ``PyUnicode_FromFormat``
+  * ctypes ``char_p`` and ``unichar_p`` indexing now CPython compatible
+  * ``gcdump`` now reports largest object
+  * More complete support in the ``_curses`` CFFI module
+  * Add cPickle.Unpickler.find_global (issue 1853_)
+  * Fix ``PyErr_Fetch`` + ``PyErr_NormalizeException`` with no exception set
+  * Simplify ``gc.get_referrers()`` to return the opposite of ``gc.get_referents()``
+  * Update RevDB to version pypy2.7-v5.6.2
+  * Previously, ``instance.method`` would return always the same bound method
+    object, when gotten from the same instance (as far as ``is`` and ``id()``
+    can tell).  CPython doesn't do that.  Now PyPy, like CPython, returns a 
+    different bound method object every time.  For ``type.method``, PyPy2 still
+    returns always the same *unbound* method object; CPython does it for built-in
+    types but not for user-defined types
+  * Link to disable PaX protection for the JIT when needed
+  * Update build instructions and an rarely used Makefile
+  * Recreate support for using leakfinder in cpyext tests which had suffered
+    bit-rot, disable due to many false positives
+  * Add more functionality to ``sysconfig``
+  * Added ``_swappedbytes_`` support for ``ctypes.Structure``
+  * Better support the ``inspect`` module on ``frames``
+
+* Bug Fixes 
+
+  * Fix issue 2592_ - cpyext ``PyListObject.pop``, ``pop_end`` must return a value
+  * Implement ``PyListOjbect.getstorage_copy``
+  * Fix for ``reversed(dictproxy)`` issue 2601_
+  * Fix for duplicate names in ctypes' ``_fields__``, issue 2621_
+  * Update built-in ``pyexpat`` module on win32 to use UTF-8 version not UTF-16
+  * ``gc.get_objects`` now handles objects with finalizers more consistently
+  * Fixed memory leak in ``SSLContext.getpeercert`` returning validated
+    certificates and ``SSLContext.get_ca_certs(binary_mode=True)``
+    (_get_crl_dp) `CPython issue 29738`_
+
+* Performance improvements:
+
+  * Improve performance of ``bytearray.extend`` by rewriting portions in app-level
+  * Optimize list accesses with constant indexes better by retaining more
+    information about them
+  * Add a jit driver for ``array.count`` and ``array.index``
+  * Improve information retained in a bridge wrt ``array``
+  * Move some dummy CAPI functions and ``Py*_Check`` functions from RPython into
+    pure C macros
+  * In the fast ``zip(intlist1, intlist2)`` implementation, don't wrap and unwrap
+    all the ints
+  * Cache string keys that occur in JSON dicts, as they are likely to repeat
+
+* RPython improvements
+
+  * Do not preallocate a RPython list if we only know an upper bound on its size
+  * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
+  * Replace magical NOT RPYTHON comment with a decorator
+  * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
+  * Reduce excessive ``memory_pressure`` for ``_SSLContext`` objects and add
+    ``memory_pressure`` for ``_SSLSocket`` objects
+
+* Degredations
+
+  * Disable vmprof on win32, due to upstream changes that break the internal ``_vmprof`` module
+
+.. _here: cpython_differences.html
+.. _1853: https://bitbucket.org/pypy/pypy/issues/1853
+.. _2592: https://bitbucket.org/pypy/pypy/issues/2592
+.. _2590: https://bitbucket.org/pypy/pypy/issues/2590
+.. _2621: https://bitbucket.org/pypy/pypy/issues/2621
+
+Highlights of the PyPy3.5 release (since 5.8 beta released June 2017)
+======================================================================
+
+* New features
+
+  * Add support for ``_PyNamespace_New``, ``PyMemoryView_FromMemory``, 
+    ``Py_EnterRecursiveCall`` raising RecursionError, ``PyObject_LengthHint``,
+    ``PyUnicode_FromKindAndData``, ``PyDict_SetDefault``, ``PyGenObject``,
+    ``PyGenObject``, ``PyUnicode_Substring``, ``PyLong_FromUnicodeObject``
+  * Implement ``PyType_FromSpec`` (PEP 384) and fix issues with PEP 489 support
+  * Support the new version of ``os.stat()`` on win32
+  * Use ``stat3()`` on Posix
+  * Accept buffer objects as filenames, except for `oslistdir``
+  * Make slices of array ``memoryview`` s usable as writable buffers if contiguous
+  * Better handling of ``'%s'`` formatting for byte strings which might be utf-8 encoded
+  * Update the macros ``Py_DECREF`` and similar to use the CPython 3.5 version
+  * Ensure that ``mappingproxy`` is recognised as a mapping, not a sequence
+  * Enable PGO for CLang
+  * Rework ``cppyy`` packaging and rename the backend to ``_cppyy``
+  * Support for libressl 2.5.4
+  * Mirror CPython ``classmethod __reduce__`` which fixes pickling test
+  * Use utf-8 for ``readline`` history file
+  * Allow assigning ``'__class__'`` between ``ModuleType`` and its subclasses
+  * Add async slot functions in cpyext
+
+* Bug Fixes
+
+  * Try to make ``openssl`` CFFI bindings more general and future-proof
+  * Better support ``importlib`` by only listing built-in modules in ``sys.builtin``
+  * Add ``memory_pressure`` to large CFFI allocations in ``_lzma``, issue 2579_
+  * Fix for ``reversed(mapping object)`` issue 2601_
+  * Fixing regression with non-started generator receiving non-``None``, should
+    always raise ``TypeError``
+  * ``itertools.islice``: use same logic as CPython, fixes 2643_
+
+* Performance improvements:
+
+  * 
+
+* The following features of Python 3.5 are not implemented yet in PyPy:
+
+  * PEP 442: Safe object finalization
+
+.. _resolved: whatsnew-pypy2-5.9.0.html
+.. _2579: https://bitbucket.org/pypy/pypy/issues/2579
+.. _2601: https://bitbucket.org/pypy/pypy/issues/2601
+.. _2643: https://bitbucket.org/pypy/pypy/issues/2643
+.. _CPython issue 29738: https://bugs.python.org/issue29738
+
+Please update, and continue to help us make PyPy better.
+
+Cheers
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -1,87 +1,6 @@
-==========================
-What's new in PyPy2.7 5.9+
-==========================
-
-.. this is a revision shortly after release-pypy2.7-v5.8.0
-.. startrev: 558bd00b3dd8
-
-In previous versions of PyPy, ``instance.method`` would return always
-the same bound method object, when gotten out of the same instance (as
-far as ``is`` and ``id()`` can tell).  CPython doesn't do that.  Now
-PyPy, like CPython, returns a different bound method object every time.
-For ``type.method``, PyPy2 still returns always the same *unbound*
-method object; CPython does it for built-in types but not for
-user-defined types.
-
-.. branch: cffi-complex
-.. branch: cffi-char16-char32
-
-The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
-
-.. branch: ctypes_char_indexing
-
-Indexing into char* behaves differently than CPython
-
-.. branch: vmprof-0.4.8
-
-Improve and fix issues with vmprof
-
-.. branch: issue-2592
-
-CPyext PyListObject.pop must return the value
-
-.. branch: cpyext-hash_notimpl
-
-If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None
-
-.. branch: cppyy-packaging
-
-Renaming of ``cppyy`` to ``_cppyy``.
-The former is now an external package installable with ``pip install cppyy``.
-
-.. branch: Enable_PGO_for_clang
-
-.. branch: nopax
-
-At the end of translation, run ``attr -q -s pax.flags -V m`` on
-PAX-enabled systems on the produced binary.  This seems necessary
-because PyPy uses a JIT.
-
-.. branch: pypy_bytearray
-
-Improve ``bytearray`` performance (backported from py3.5)
-
-.. branch: gc-del-limit-growth
-
-Fix the bounds in the GC when allocating a lot of objects with finalizers,
-fixes issue #2590
-
-.. branch: arrays-force-less
-
-Small improvement to optimize list accesses with constant indexes better by
-throwing away information about them less eagerly.
-
-
-.. branch: getarrayitem-into-bridges
-
-More information is retained into a bridge: knowledge about the content of
-arrays (at fixed indices) is stored in guards (and thus available at the
-beginning of bridges). Also, some better feeding of information about known
-fields of constant objects into bridges.
-
-.. branch: cpyext-leakchecking
-
-Add support for leakfinder in cpyext tests (disabled for now, due to too many
-failures).
-
-.. branch: pypy_swappedbytes
-
-Added ``_swappedbytes_`` support for ``ctypes.Structure``
-
-.. branch: pycheck-macros
-
-Convert many Py*_Check cpyext functions into macros, like CPython.
-
-.. branch: py_ssize_t
-
-Explicitly use Py_ssize_t as the Signed type in pypy c-api
+===========================
+What's new in PyPy2.7 5.10+
+===========================
+
+.. this is a revision shortly after release-pypy2.7-v5.9.0
+.. startrev:899e5245de1e
diff --git a/pypy/doc/whatsnew-pypy2-5.9.0.rst b/pypy/doc/whatsnew-pypy2-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy2-5.9.0.rst
@@ -0,0 +1,87 @@
+=========================
+What's new in PyPy2.7 5.9
+=========================
+
+.. this is a revision shortly after release-pypy2.7-v5.8.0
+.. startrev: 558bd00b3dd8
+
+In previous versions of PyPy, ``instance.method`` would return always
+the same bound method object, when gotten out of the same instance (as
+far as ``is`` and ``id()`` can tell).  CPython doesn't do that.  Now
+PyPy, like CPython, returns a different bound method object every time.
+For ``type.method``, PyPy2 still returns always the same *unbound*
+method object; CPython does it for built-in types but not for
+user-defined types.
+
+.. branch: cffi-complex
+.. branch: cffi-char16-char32
+
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
+
+.. branch: ctypes_char_indexing
+
+Indexing into char* behaves differently than CPython
+
+.. branch: vmprof-0.4.8
+
+Improve and fix issues with vmprof
+
+.. branch: issue-2592
+
+CPyext PyListObject.pop must return the value
+
+.. branch: cpyext-hash_notimpl
+
+If ``tp_hash`` is ``PyObject_HashNotImplemented``, set ``obj.__dict__['__hash__']`` to None
+
+.. branch: cppyy-packaging
+
+Renaming of ``cppyy`` to ``_cppyy``.
+The former is now an external package installable with ``pip install cppyy``.
+
+.. branch: Enable_PGO_for_clang
+
+.. branch: nopax
+
+At the end of translation, run ``attr -q -s pax.flags -V m`` on
+PAX-enabled systems on the produced binary.  This seems necessary
+because PyPy uses a JIT.
+
+.. branch: pypy_bytearray
+
+Improve ``bytearray`` performance (backported from py3.5)
+
+.. branch: gc-del-limit-growth
+
+Fix the bounds in the GC when allocating a lot of objects with finalizers,
+fixes issue #2590
+
+.. branch: arrays-force-less
+
+Small improvement to optimize list accesses with constant indexes better by
+throwing away information about them less eagerly.
+
+
+.. branch: getarrayitem-into-bridges
+
+More information is retained into a bridge: knowledge about the content of
+arrays (at fixed indices) is stored in guards (and thus available at the
+beginning of bridges). Also, some better feeding of information about known
+fields of constant objects into bridges.
+
+.. branch: cpyext-leakchecking
+
+Add support for leakfinder in cpyext tests (disabled for now, due to too many
+failures).
+
+.. branch: pypy_swappedbytes
+
+Added ``_swappedbytes_`` support for ``ctypes.Structure``
+
+.. branch: pycheck-macros
+
+Convert many Py*_Check cpyext functions into macros, like CPython.
+
+.. branch: py_ssize_t
+
+Explicitly use Py_ssize_t as the Signed type in pypy c-api
diff --git a/pypy/doc/whatsnew-pypy3-5.9.0.rst b/pypy/doc/whatsnew-pypy3-5.9.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.9.0.rst
@@ -0,0 +1,7 @@
+=======================
+What's new in PyPy3 5.9
+=======================
+
+.. this is the revision after release-pypy3.5-5.8
+.. startrev: afbf09453369
+
diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst
--- a/pypy/doc/whatsnew-pypy3-head.rst
+++ b/pypy/doc/whatsnew-pypy3-head.rst
@@ -1,9 +1,9 @@
 =========================
-What's new in PyPy3 5.8+
+What's new in PyPy3 5.9+
 =========================
 
-.. this is the revision after release-pypy3.3-5.8.x was branched
-.. startrev: c173df164527
+.. this is the revision after release-pypy3.5-5.9
+.. startrev: be41e3ac0a29
 
 .. branch: multiphase
 
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.11.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -1,6 +1,6 @@
 import sys
 from rpython.rlib.rstring import StringBuilder
-from rpython.rlib.objectmodel import specialize, always_inline
+from rpython.rlib.objectmodel import specialize, always_inline, r_dict
 from rpython.rlib import rfloat, runicode
 from rpython.rtyper.lltypesystem import lltype, rffi
 from pypy.interpreter.error import oefmt, OperationError
@@ -42,6 +42,22 @@
         ll_res.chars[i] = cast_primitive(UniChar, ch)
     return hlunicode(ll_res)
 
+def slice_eq(a, b):
+    (ll_chars1, start1, length1, _) = a
+    (ll_chars2, start2, length2, _) = b
+    if length1 != length2:
+        return False
+    j = start2
+    for i in range(start1, start1 + length1):
+        if ll_chars1[i] != ll_chars2[j]:
+            return False
+        j += 1
+    return True
+
+def slice_hash(a):
+    (ll_chars, start, length, h) = a
+    return h
+
 class DecoderError(Exception):
     def __init__(self, msg, pos):
         self.msg = msg
@@ -60,8 +76,7 @@
         self.ll_chars = rffi.str2charp(s)
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
-        self.last_type = TYPE_UNKNOWN
-        self.memo = {}
+        self.cache = r_dict(slice_eq, slice_hash)
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -249,21 +264,16 @@
 
     def decode_object(self, i):
         start = i
-        w_dict = self.space.newdict()
-        #
+
         i = self.skip_whitespace(i)
         if self.ll_chars[i] == '}':
             self.pos = i+1
-            return w_dict
-        #
+            return self.space.newdict()
+
+        d = {}
         while True:
             # parse a key: value
-            self.last_type = TYPE_UNKNOWN
-            w_name = self.decode_any(i)
-            if self.last_type != TYPE_STRING:
-                raise DecoderError("Key name must be string for object starting at", start)
-            w_name = self.memo.setdefault(self.space.unicode_w(w_name), w_name)
-
+            name = self.decode_key(i)
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
@@ -272,13 +282,13 @@
             i = self.skip_whitespace(i)
             #
             w_value = self.decode_any(i)
-            self.space.setitem(w_dict, w_name, w_value)
+            d[name] = w_value
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             i += 1
             if ch == '}':
                 self.pos = i
-                return w_dict
+                return self._create_dict(d)
             elif ch == ',':
                 pass
             elif ch == '\0':
@@ -287,6 +297,9 @@
                 raise DecoderError("Unexpected '%s' when decoding object" % ch,
                                    i-1)
 
+    def _create_dict(self, d):
+        from pypy.objspace.std.dictmultiobject import from_unicode_key_dict
+        return from_unicode_key_dict(self.space, d)
 
     def decode_string(self, i):
         start = i
@@ -298,22 +311,23 @@
             i += 1
             bits |= ord(ch)
             if ch == '"':
-                if bits & 0x80:
-                    # the 8th bit is set, it's an utf8 strnig
-                    content_utf8 = self.getslice(start, i-1)
-                    content_unicode = unicodehelper.decode_utf8(self.space, content_utf8)
-                else:
-                    # ascii only, fast path (ascii is a strict subset of
-                    # latin1, and we already checked that all the chars are <
-                    # 128)
-                    content_unicode = strslice2unicode_latin1(self.s, start, i-1)
-                self.last_type = TYPE_STRING
                 self.pos = i
-                return self.space.newunicode(content_unicode)
+                return self.space.newunicode(
+                        self._create_string(start, i - 1, bits))
             elif ch == '\\' or ch < '\x20':
                 self.pos = i-1
                 return self.decode_string_escaped(start)
 
+    def _create_string(self, start, end, bits):
+        if bits & 0x80:
+            # the 8th bit is set, it's an utf8 string
+            content_utf8 = self.getslice(start, end)
+            return unicodehelper.decode_utf8(self.space, content_utf8)
+        else:
+            # ascii only, fast path (ascii is a strict subset of
+            # latin1, and we already checked that all the chars are <
+            # 128)
+            return strslice2unicode_latin1(self.s, start, end)
 
     def decode_string_escaped(self, start):
         i = self.pos
@@ -326,9 +340,7 @@
             i += 1
             if ch == '"':
                 content_utf8 = builder.build()
-                content_unicode = unicodehelper.decode_utf8(
-                    self.space, content_utf8, allow_surrogates=True)
-                self.last_type = TYPE_STRING
+                content_unicode = unicodehelper.decode_utf8(self.space, content_utf8, allow_surrogates=True)
                 self.pos = i
                 return self.space.newunicode(content_unicode)
             elif ch == '\\':
@@ -390,6 +402,47 @@
         lowsurr = int(hexdigits, 16) # the possible ValueError is caugth by the caller
         return 0x10000 + (((highsurr - 0xd800) << 10) | (lowsurr - 0xdc00))
 
+    def decode_key(self, i):
+        """ returns an unwrapped unicode """
+        from rpython.rlib.rarithmetic import intmask
+
+        i = self.skip_whitespace(i)
+        ll_chars = self.ll_chars
+        ch = ll_chars[i]
+        if ch != '"':
+            raise DecoderError("Key name must be string at char", i)
+        i += 1
+
+        start = i
+        bits = 0
+        strhash = ord(ll_chars[i]) << 7
+        while True:
+            ch = ll_chars[i]
+            i += 1
+            if ch == '"':
+                break
+            elif ch == '\\' or ch < '\x20':
+                self.pos = i-1
+                return self.space.unicode_w(self.decode_string_escaped(start))
+            strhash = intmask((1000003 * strhash) ^ ord(ll_chars[i]))
+            bits |= ord(ch)
+        length = i - start - 1
+        if length == 0:
+            strhash = -1
+        else:
+            strhash ^= length
+            strhash = intmask(strhash)
+        self.pos = i
+        # check cache first:
+        key = (ll_chars, start, length, strhash)
+        try:
+            return self.cache[key]
+        except KeyError:
+            pass
+        res = self._create_string(start, i - 1, bits)
+        self.cache[key] = res
+        return res
+
 def loads(space, w_s, w_errorcls=None):
     s = space.text_w(w_s)
     decoder = JSONDecoder(space, s)
diff --git a/pypy/module/_pypyjson/targetjson.py b/pypy/module/_pypyjson/targetjson.py
--- a/pypy/module/_pypyjson/targetjson.py
+++ b/pypy/module/_pypyjson/targetjson.py
@@ -5,9 +5,15 @@
 
 import time
 from pypy.interpreter.error import OperationError
-from pypy.module._pypyjson.interp_decoder import loads
+from pypy.module._pypyjson.interp_decoder import loads, JSONDecoder
 from rpython.rlib.objectmodel import specialize, dont_inline
 
+def _create_dict(self, d):
+    w_res = W_Dict()
+    w_res.dictval = d
+    return w_res
+
+JSONDecoder._create_dict = _create_dict
 
 ## MSG = open('msg.json').read()
 
@@ -65,10 +71,14 @@
     def isinstance_w(self, w_x, w_type):
         return isinstance(w_x, w_type)
 
-    def str_w(self, w_x):
+    def bytes_w(self, w_x):
         assert isinstance(w_x, W_String)
         return w_x.strval
 
+    def unicode_w(self, w_x):
+        assert isinstance(w_x, W_Unicode)
+        return w_x.unival
+
     @dont_inline
     def call_method(self, obj, name, arg):
         assert name == 'append'
@@ -83,13 +93,17 @@
         assert isinstance(key, W_Unicode)
         d.dictval[key.unival] = value
 
-    def wrapunicode(self, x):
+    def newunicode(self, x):
         return W_Unicode(x)
 
-    def wrapint(self, x):
+    def newtext(self, x):
+        return W_String(x)
+    newbytes = newtext
+
+    def newint(self, x):
         return W_Int(x)
 
-    def wrapfloat(self, x):
+    def newfloat(self, x):
         return W_Float(x)
 
     @specialize.argtype(1)
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -10,7 +10,18 @@
     assert dec.skip_whitespace(8) == len(s)
     dec.close()
 
-
+def test_decode_key():
+    s1 = "123" * 100
+    s = ' "%s"   "%s" ' % (s1, s1)
+    dec = JSONDecoder('fake space', s)
+    assert dec.pos == 0
+    x = dec.decode_key(0)
+    assert x == s1
+    # check caching
+    y = dec.decode_key(dec.pos)
+    assert y == s1
+    assert y is x
+    dec.close()
 
 class AppTest(object):
     spaceconfig = {"usemodules": ['_pypyjson']}
@@ -189,6 +200,12 @@
         res = _pypyjson.loads(json)
         assert res == {u'a': u'\ud83d'}
 
+    def test_cache_keys(self):
+        import _pypyjson
+        json = '[{"a": 1}, {"a": 2}]'
+        res = _pypyjson.loads(json)
+        assert res == [{u'a': 1}, {u'a': 2}]
+
     def test_tab_in_string_should_fail(self):
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
@@ -226,7 +243,7 @@
             ('{"spam":[42}', "Unexpected '}' when decoding array", 11),
             ('["]', 'Unterminated string starting at', 1),
             ('["spam":', "Unexpected ':' when decoding array", 7),
-            ('[{]', "Unexpected ']' at", 2),
+            ('[{]', "Key name must be string at char", 2),
         ]
         for inputtext, errmsg, errpos in test_cases:
             exc = raises(ValueError, _pypyjson.loads, inputtext)
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -562,6 +562,7 @@
     '_PyObject_CallFunction_SizeT', '_PyObject_CallMethod_SizeT',
 
     'PyObject_GetBuffer', 'PyBuffer_Release',
+    '_Py_setfilesystemdefaultencoding',
 
     'PyCObject_FromVoidPtr', 'PyCObject_FromVoidPtrAndDesc', 'PyCObject_AsVoidPtr',
     'PyCObject_GetDesc', 'PyCObject_Import', 'PyCObject_SetVoidPtr',
@@ -1058,10 +1059,16 @@
     get_capsule_type = rffi.llexternal('_%s_get_capsule_type' % prefix,
                                        [], PyTypeObjectPtr,
                                        compilation_info=eci, _nowrapper=True)
+    setdefenc = rffi.llexternal('_%s_setfilesystemdefaultencoding' % prefix,
+                                [rffi.CCHARP], lltype.Void,
+                                compilation_info=eci, _nowrapper=True)
     def init_types(space):
         from pypy.module.cpyext.typeobject import py_type_ready
+        from pypy.module.sys.interp_encoding import getfilesystemencoding
         py_type_ready(space, get_cobject_type())
         py_type_ready(space, get_capsule_type())
+        s = space.text_w(getfilesystemencoding(space))
+        setdefenc(rffi.str2charp(s, track_allocation=False))  # "leaks"
     INIT_FUNCTIONS.append(init_types)
     from pypy.module.posix.interp_posix import add_fork_hook
     global py_fatalerror
@@ -1330,6 +1337,18 @@
     decls = defaultdict(list)
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
+    decls[pypy_decl].append("""
+        /* hack for https://bugs.python.org/issue29943 */
+        PyAPI_FUNC(int) %s(PySliceObject *arg0,
+                           Signed arg1, Signed *arg2,
+                           Signed *arg3, Signed *arg4, Signed *arg5);
+        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+                Py_ssize_t *arg5) {
+            return %s(arg0, arg1, arg2, arg3,
+                      arg4, arg5);
+        }
+    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]
diff --git a/pypy/module/cpyext/include/fileobject.h b/pypy/module/cpyext/include/fileobject.h
--- a/pypy/module/cpyext/include/fileobject.h
+++ b/pypy/module/cpyext/include/fileobject.h
@@ -1,1 +1,2 @@
-#define Py_FileSystemDefaultEncoding   NULL
+PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding;
+PyAPI_FUNC(void) _Py_setfilesystemdefaultencoding(const char *);
diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h
--- a/pypy/module/cpyext/include/patchlevel.h
+++ b/pypy/module/cpyext/include/patchlevel.h
@@ -30,7 +30,7 @@
 
 /* PyPy version as a string */
 #define PYPY_VERSION "5.10.0-alpha0"
-#define PYPY_VERSION_NUM  0x05100000
+#define PYPY_VERSION_NUM  0x050A0000
 
 /* Defined to mean a PyPy where cpyext holds more regular references
    to PyObjects, e.g. staying alive as long as the internal PyPy object
diff --git a/pypy/module/cpyext/src/missing.c b/pypy/module/cpyext/src/missing.c
--- a/pypy/module/cpyext/src/missing.c
+++ b/pypy/module/cpyext/src/missing.c
@@ -27,3 +27,7 @@
 int Py_Py3kWarningFlag = 0;
 int Py_HashRandomizationFlag = 0;
 
+const char *Py_FileSystemDefaultEncoding;  /* filled when cpyext is imported */
+void _Py_setfilesystemdefaultencoding(const char *enc) {
+    Py_FileSystemDefaultEncoding = enc;
+}
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test/test_fileobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_fileobject.py
@@ -0,0 +1,13 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+
+class AppTestFileObject(AppTestCpythonExtensionBase):
+    def test_defaultencoding(self):
+        import sys
+        module = self.import_extension('foo', [
+            ("defenc", "METH_NOARGS",
+             """
+                return PyString_FromString(Py_FileSystemDefaultEncoding);
+             """),
+            ])
+        assert module.defenc() == sys.getfilesystemencoding()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -2498,8 +2498,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2517,8 +2517,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -2337,8 +2337,8 @@
     for i in range(2000):
         p = lib.malloc(50*1024*1024)    # 50 MB
         p1 = ffi.cast("char *", p)
-        for j in xrange(0, 50*1024*1024, 4096):
-            p1[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p1[j] = b'!'
         p = ffi.gc(p, lib.free, 50*1024*1024)
         x = X()
         x.p = p
@@ -2356,8 +2356,8 @@
         pass
     for i in range(2000):
         p = ffi.new("char[]", 50*1024*1024)    # 50 MB
-        for j in xrange(0, 50*1024*1024, 4096):
-            p[j] = '!'
+        for j in range(0, 50*1024*1024, 4096):
+            p[j] = b'!'
         x = X()
         x.p = p
         x.cyclic = x
diff --git a/pypy/objspace/std/dictmultiobject.py b/pypy/objspace/std/dictmultiobject.py
--- a/pypy/objspace/std/dictmultiobject.py
+++ b/pypy/objspace/std/dictmultiobject.py
@@ -1232,6 +1232,12 @@
 create_iterator_classes(UnicodeDictStrategy)
 
 
+def from_unicode_key_dict(space, d):
+    strategy = space.fromcache(UnicodeDictStrategy)
+    storage = strategy.erase(d)
+    return W_DictObject(space, strategy, storage)
+
+
 class IntDictStrategy(AbstractTypedStrategy, DictStrategy):
     erase, unerase = rerased.new_erasing_pair("int")
     erase = staticmethod(erase)

From pypy.commits at gmail.com  Thu Sep 28 16:24:49 2017
From: pypy.commits at gmail.com (mjacob)
Date: Thu, 28 Sep 2017 13:24:49 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge py3.5
Message-ID: <59cd5a91.cf97df0a.89a58.6182@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.6
Changeset: r92492:2a061e98f19f
Date: 2017-09-28 22:24 +0200
http://bitbucket.org/pypy/pypy/changeset/2a061e98f19f/

Log:	hg merge py3.5

diff too long, truncating to 2000 out of 33754 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -27,16 +27,17 @@
 ^pypy/module/cpyext/test/.+\.manifest$
 ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^pypy/module/cppyy/src/.+\.o$
-^pypy/module/cppyy/bench/.+\.so$
-^pypy/module/cppyy/bench/.+\.root$
-^pypy/module/cppyy/bench/.+\.d$
-^pypy/module/cppyy/src/.+\.errors$
-^pypy/module/cppyy/test/.+_rflx\.cpp$
-^pypy/module/cppyy/test/.+\.so$
-^pypy/module/cppyy/test/.+\.rootmap$
-^pypy/module/cppyy/test/.+\.exe$
-^pypy/module/cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/src/.+\.o$
+^pypy/module/_cppyy/bench/.+\.so$
+^pypy/module/_cppyy/bench/.+\.root$
+^pypy/module/_cppyy/bench/.+\.d$
+^pypy/module/_cppyy/src/.+\.errors$
+^pypy/module/_cppyy/test/.+_rflx\.cpp$
+^pypy/module/_cppyy/test/.+\.so$
+^pypy/module/_cppyy/test/.+\.rootmap$
+^pypy/module/_cppyy/test/.+\.exe$
+^pypy/module/_cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/.+/*\.pcm$
 ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
 ^pypy/doc/.+\.html$
 ^pypy/doc/config/.+\.rst$
@@ -93,6 +94,3 @@
 ^release/
 ^rpython/_cache$
 
-pypy/module/cppyy/.+/*\.pcm
-
-
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@
 RUNINTERP = $(PYPY_EXECUTABLE)
 endif
 
-.PHONY: cffi_imports
+.PHONY: pypy-c cffi_imports
 
 pypy-c:
 	@echo
@@ -32,7 +32,7 @@
 	@echo "===================================================================="
 	@echo
 	@sleep 5
-	$(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+	cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit targetpypystandalone.py
 
 # Note: the -jN option, or MAKEFLAGS=-jN, are not usable.  They are
 # replaced with an opaque --jobserver option by the time this Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
+	PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -361,17 +361,20 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+            if _sys.maxint > 2 ** 32:
+                handle = int(handle)   # long -> int
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxint * 2 + 1))
-
+        return "<%s '%s', handle %x at %x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxint*2 + 1)),
+                id(self) & (_sys.maxint*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -218,6 +218,10 @@
 
         compiler.shared_lib_extension = so_ext
 
+def get_config_h_filename():
+    """Returns the path of pyconfig.h."""
+    inc_dir = get_python_inc(plat_specific=1)
+    return os.path.join(inc_dir, 'pyconfig.h')
 
 from sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
         return "-L" + dir
 
     def _is_gcc(self, compiler_name):
-        return "gcc" in compiler_name or "g++" in compiler_name
+        # XXX PyPy workaround, look at the big comment below for more
+        # context. On CPython, the hack below works fine because
+        # `compiler_name` contains the name of the actual compiler which was
+        # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+        # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+        # result is that we pass the wrong option to the compiler.
+        #
+        # The workaround is to *always* pretend to be GCC if we are on Linux:
+        # this should cover the vast majority of real systems, including the
+        # ones which use clang (which understands the '-Wl,-rpath' syntax as
+        # well)
+        return (sys.platform == "linux2" or
+                "gcc" in compiler_name or "g++" in compiler_name)
 
     def runtime_library_dir_option(self, dir):
         # XXX Hackish, at the very least.  See Python bug #445902:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except (AttributeError, StopIteration):
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 
diff --git a/lib-python/3/ctypes/__init__.py b/lib-python/3/ctypes/__init__.py
--- a/lib-python/3/ctypes/__init__.py
+++ b/lib-python/3/ctypes/__init__.py
@@ -347,16 +347,18 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxsize * 2 + 1))
+        return "<%s '%s', handle %x at 0x%x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxsize*2 + 1)),
+                id(self) & (_sys.maxsize*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/3/ctypes/test/test_byteswap.py b/lib-python/3/ctypes/test/test_byteswap.py
--- a/lib-python/3/ctypes/test/test_byteswap.py
+++ b/lib-python/3/ctypes/test/test_byteswap.py
@@ -2,7 +2,6 @@
 from binascii import hexlify
 
 from ctypes import *
-from ctypes.test import xfail
 
 def bin(s):
     return hexlify(memoryview(s)).decode().upper()
@@ -43,7 +42,6 @@
         with self.assertRaises(AttributeError):
             little.z = 24
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -71,7 +69,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -100,7 +97,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -129,7 +125,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -148,7 +143,6 @@
         self.assertAlmostEqual(s.value, math.pi, places=6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -176,7 +170,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -212,7 +205,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -241,7 +233,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -265,7 +256,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -827,7 +827,8 @@
             month = self._month
         if day is None:
             day = self._day
-        return date(year, month, day)
+        # PyPy fix: returns type(self)() instead of date()
+        return type(self)(year, month, day)
 
     # Comparisons of date objects with other.
 
@@ -1320,7 +1321,8 @@
             tzinfo = self.tzinfo
         if fold is None:
             fold = self._fold
-        return time(hour, minute, second, microsecond, tzinfo, fold=fold)
+        # PyPy fix: returns type(self)() instead of time()
+        return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold)
 
     # Pickle support.
 
@@ -1601,7 +1603,8 @@
             tzinfo = self.tzinfo
         if fold is None:
             fold = self.fold
-        return datetime(year, month, day, hour, minute, second,
+        # PyPy fix: returns type(self)() instead of datetime()
+        return type(self)(year, month, day, hour, minute, second,
                           microsecond, tzinfo, fold=fold)
 
     def _local_timezone(self):
diff --git a/lib-python/3/distutils/sysconfig_pypy.py b/lib-python/3/distutils/sysconfig_pypy.py
--- a/lib-python/3/distutils/sysconfig_pypy.py
+++ b/lib-python/3/distutils/sysconfig_pypy.py
@@ -73,7 +73,7 @@
     g['CCSHARED'] = "-fPIC"
     g['LDSHARED'] = "cc -pthread -shared"
     g['EXT_SUFFIX'] = so_ext
-    g['SHLIB_SUFFIX'] = so_ext
+    g['SHLIB_SUFFIX'] = ".so"
     g['SO'] = so_ext  # deprecated in Python 3, for backward compatibility
     g['AR'] = "ar"
     g['ARFLAGS'] = "rc"
@@ -81,6 +81,19 @@
     g['LIBDIR'] = os.path.join(sys.prefix, 'lib')
     g['VERSION'] = get_python_version()
 
+    if sys.platform[:6] == "darwin":
+        import platform
+        if platform.machine() == 'i386':
+            if platform.architecture()[0] == '32bit':
+                arch = 'i386'
+            else:
+                arch = 'x86_64'
+        else:
+            # just a guess
+            arch = platform.machine()
+        g['LDSHARED'] += ' -undefined dynamic_lookup'
+        g['CC'] += ' -arch %s' % (arch,)
+
     global _config_vars
     _config_vars = g
 
diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py
--- a/lib-python/3/stat.py
+++ b/lib-python/3/stat.py
@@ -139,13 +139,21 @@
 def filemode(mode):
     """Convert a file's mode to a string of the form '-rwxrwxrwx'."""
     perm = []
+
+    # The first group gets a question mark if none of the bits match the mode.
+    empty = "?"
+
     for table in _filemode_table:
         for bit, char in table:
             if mode & bit == bit:
                 perm.append(char)
                 break
         else:
-            perm.append("-")
+            perm.append(empty)
+
+        # All the rest of the positions get a - if the bits don't match.
+        empty = "-"
+
     return "".join(perm)
 
 
diff --git a/lib-python/3/test/test_asyncio/test_base_events.py b/lib-python/3/test/test_asyncio/test_base_events.py
--- a/lib-python/3/test/test_asyncio/test_base_events.py
+++ b/lib-python/3/test/test_asyncio/test_base_events.py
@@ -1588,9 +1588,15 @@
                 sock.getsockopt(
                     socket.SOL_SOCKET, socket.SO_REUSEADDR))
         if reuseport_supported:
-            self.assertFalse(
-                sock.getsockopt(
-                    socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            try:
+                self.assertFalse(
+                    sock.getsockopt(
+                        socket.SOL_SOCKET, socket.SO_REUSEPORT))
+            except OSError:
+                # Python's socket module was compiled using modern headers
+                # thus defining SO_REUSEPORT but this process is running
+                # under an older kernel that does not support SO_REUSEPORT.
+                reuseport_supported = False
         self.assertFalse(
             sock.getsockopt(
                 socket.SOL_SOCKET, socket.SO_BROADCAST))
diff --git a/lib-python/3/test/test_importlib/builtin/test_loader.py b/lib-python/3/test/test_importlib/builtin/test_loader.py
--- a/lib-python/3/test/test_importlib/builtin/test_loader.py
+++ b/lib-python/3/test/test_importlib/builtin/test_loader.py
@@ -1,6 +1,8 @@
 from .. import abc
 from .. import util
 
+from importlib.machinery import BuiltinImporter
+
 machinery = util.import_importlib('importlib.machinery')
 
 import sys
@@ -14,7 +16,7 @@
 
     def setUp(self):
         self.verification = {'__name__': 'errno', '__package__': '',
-                             '__loader__': self.machinery.BuiltinImporter}
+                             '__loader__': BuiltinImporter}  # PyPy change
 
     def verify(self, module):
         """Verify that the module matches against what it should have."""
diff --git a/lib-python/3/test/test_importlib/extension/test_loader.py b/lib-python/3/test/test_importlib/extension/test_loader.py
--- a/lib-python/3/test/test_importlib/extension/test_loader.py
+++ b/lib-python/3/test/test_importlib/extension/test_loader.py
@@ -88,6 +88,7 @@
 
     def setUp(self):
         self.name = '_testmultiphase'
+        __import__(self.name)  # PyPy hack
         finder = self.machinery.FileFinder(None)
         self.spec = importlib.util.find_spec(self.name)
         assert self.spec
@@ -145,7 +146,8 @@
             importlib.reload(module)
             self.assertIs(ex_class, module.Example)
 
-    def test_try_registration(self):
+    # XXX: PyPy doesn't support the PyState_* functions yet
+    def XXXtest_try_registration(self):
         '''Assert that the PyState_{Find,Add,Remove}Module C API doesn't work'''
         module = self.load_module()
         with self.subTest('PyState_FindModule'):
diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -278,6 +278,11 @@
                 if n is not None and n > 4:
                     n += 10**6
                 return n
+            def read(self, n):   # PyPy calls read(), not readinto()
+                result = super().read(n)
+                if len(result) > 4:
+                    result += b'\x00' * (10**6)
+                return result
         for value in (1.0, 1j, b'0123456789', '0123456789'):
             self.assertRaises(ValueError, marshal.load,
                               BadReader(marshal.dumps(value)))
@@ -355,7 +360,8 @@
     strobj = "abcde"*3
     dictobj = {"hello":floatobj, "goodbye":floatobj, floatobj:"hello"}
 
-    def helper3(self, rsample, recursive=False, simple=False):
+    def helper3(self, rsample, recursive=False, simple=False,
+                check_sharing=True, check_non_sharing=True):
         #we have two instances
         sample = (rsample, rsample)
 
@@ -365,28 +371,35 @@
         n3 = CollectObjectIDs(set(), marshal.loads(s3))
 
         #same number of instances generated
-        self.assertEqual(n3, n0)
+        # except in one corner case on top of pypy, for code objects
+        if check_sharing:
+            self.assertEqual(n3, n0)
 
         if not recursive:
             #can compare with version 2
             s2 = marshal.dumps(sample, 2)
             n2 = CollectObjectIDs(set(), marshal.loads(s2))
             #old format generated more instances
-            self.assertGreater(n2, n0)
+            # except on pypy where equal ints or floats always have
+            # the same id anyway
+            if check_non_sharing:
+                self.assertGreater(n2, n0)
 
             #if complex objects are in there, old format is larger
-            if not simple:
+            if check_non_sharing and not simple:
                 self.assertGreater(len(s2), len(s3))
             else:
                 self.assertGreaterEqual(len(s2), len(s3))
 
     def testInt(self):
         self.helper(self.intobj)
-        self.helper3(self.intobj, simple=True)
+        self.helper3(self.intobj, simple=True,
+                     check_non_sharing=support.check_impl_detail())
 
     def testFloat(self):
         self.helper(self.floatobj)
-        self.helper3(self.floatobj)
+        self.helper3(self.floatobj,
+                     check_non_sharing=support.check_impl_detail())
 
     def testStr(self):
         self.helper(self.strobj)
@@ -402,7 +415,7 @@
         if __file__.endswith(".py"):
             code = compile(code, __file__, "exec")
         self.helper(code)
-        self.helper3(code)
+        self.helper3(code, check_sharing=support.check_impl_detail())
 
     def testRecursion(self):
         d = dict(self.dictobj)
diff --git a/lib-python/3/test/test_pyexpat.py b/lib-python/3/test/test_pyexpat.py
--- a/lib-python/3/test/test_pyexpat.py
+++ b/lib-python/3/test/test_pyexpat.py
@@ -11,7 +11,7 @@
 from xml.parsers import expat
 from xml.parsers.expat import errors
 
-from test.support import sortdict
+from test.support import sortdict, impl_detail
 
 
 class SetAttributeTest(unittest.TestCase):
@@ -446,6 +446,7 @@
         self.assertEqual(os.path.basename(entry[0]), filename)
         self.assertEqual(entry[2], funcname)
 
+    @impl_detail("PyPy does not have pyexpat.c", pypy=False)
     def test_exception(self):
         parser = expat.ParserCreate()
         parser.StartElementHandler = self.StartElementHandler
diff --git a/lib-python/3/test/test_stat.py b/lib-python/3/test/test_stat.py
--- a/lib-python/3/test/test_stat.py
+++ b/lib-python/3/test/test_stat.py
@@ -138,6 +138,10 @@
             self.assertS_IS("REG", st_mode)
             self.assertEqual(modestr, '-r--r--r--')
             self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444)
+
+            # If there are only permission bits, no type bytes, a question
+            # mark is rendered in the type field.
+            self.assertEqual(self.statmod.filemode(0o420), '?r---w----')
         else:
             os.chmod(TESTFN, 0o700)
             st_mode, modestr = self.get_mode()
diff --git a/lib-python/3/test/test_sysconfig.py b/lib-python/3/test/test_sysconfig.py
--- a/lib-python/3/test/test_sysconfig.py
+++ b/lib-python/3/test/test_sysconfig.py
@@ -401,9 +401,16 @@
             self.assertTrue('linux' in suffix, suffix)
         if re.match('(i[3-6]86|x86_64)$', machine):
             if ctypes.sizeof(ctypes.c_char_p()) == 4:
-                self.assertTrue(suffix.endswith('i386-linux-gnu.so') or
-                                suffix.endswith('x86_64-linux-gnux32.so'),
-                                suffix)
+                self.assertTrue(
+                    suffix.endswith((
+                        'i386-linux-gnu.so',
+                        'i486-linux-gnu.so',
+                        'i586-linux-gnu.so',
+                        'i686-linux-gnu.so',
+                        'x86_64-linux-gnux32.so',
+                    )),
+                    suffix,
+                )
             else: # 8 byte pointer size
                 self.assertTrue(suffix.endswith('x86_64-linux-gnu.so'), suffix)
 
diff --git a/lib_pypy/_cffi_ssl/_stdssl/certificate.py b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
--- a/lib_pypy/_cffi_ssl/_stdssl/certificate.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/certificate.py
@@ -173,14 +173,13 @@
 
     return tuple(dn)
 
-STATIC_BIO_BUF = ffi.new("char[]", 2048)
-
 def _bio_get_str(biobuf):
-    length = lib.BIO_gets(biobuf, STATIC_BIO_BUF, len(STATIC_BIO_BUF)-1)
+    bio_buf = ffi.new("char[]", 2048)
+    length = lib.BIO_gets(biobuf, bio_buf, len(bio_buf)-1)
     if length < 0:
         if biobuf: lib.BIO_free(biobuf)
         raise ssl_error(None)
-    return _str_with_len(STATIC_BIO_BUF, length)
+    return _str_with_len(bio_buf, length)
 
 def _decode_certificate(certificate):
     retval = {}
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -82,7 +82,7 @@
         return False
 
     def in_dll(self, dll, name):
-        return self.from_address(dll._handle.getaddressindll(name))
+        return self.from_address(dll.__pypy_dll__.getaddressindll(name))
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -430,7 +430,7 @@
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
 
-        cdll = self.dll._handle
+        cdll = self.dll.__pypy_dll__
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffi_restype = restype.get_ffi_argtype()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -141,6 +141,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = memoryview(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FF) << 56) | \
+               ((value & 0x000000000000FF00) << 40) | \
+               ((value & 0x0000000000FF0000) << 24) | \
+               ((value & 0x00000000FF000000) << 8) | \
+               ((value & 0x000000FF00000000) >> 8) | \
+               ((value & 0x0000FF0000000000) >> 24) | \
+               ((value & 0x00FF000000000000) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -265,6 +313,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -40,6 +40,22 @@
         else:
             rawfields.append((f[0], f[1]._ffishape_))
 
+    # hack for duplicate field names
+    already_seen = set()
+    names1 = names
+    names = []
+    for f in names1:
+        if f not in already_seen:
+            names.append(f)
+            already_seen.add(f)
+    already_seen = set()
+    for i in reversed(range(len(rawfields))):
+        if rawfields[i][0] in already_seen:
+            rawfields[i] = (('$DUP%d$%s' % (i, rawfields[i][0]),)
+                            + rawfields[i][1:])
+        already_seen.add(rawfields[i][0])
+    # /hack
+
     _set_shape(self, rawfields, self._is_union)
 
     fields = {}
@@ -130,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -224,18 +241,26 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData, metaclass=StructOrUnionMeta):
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -411,7 +411,7 @@
             val = lib.mvwget_wch(self._win, *args, wch)
         else:
             raise error("get_wch requires 0 or 2 arguments")
-        _check_ERR(val, "get_wch"):
+        _check_ERR(val, "get_wch")
         return wch[0]
 
     def getkey(self, *args):
diff --git a/lib_pypy/_testmultiphase.c b/lib_pypy/_testmultiphase.c
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_testmultiphase.c
@@ -0,0 +1,627 @@
+/* Copied from CPython's Modules/_testmultiphase.c */
+/***************************************************/
+
+/* Testing module for multi-phase initialization of extension modules (PEP 489)
+ */
+
+#include "Python.h"
+
+/* Example objects */
+typedef struct {
+    PyObject_HEAD
+    PyObject            *x_attr;        /* Attributes dictionary */
+} ExampleObject;
+
+/* Example methods */
+
+static int
+Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->x_attr);
+    return 0;
+}
+
+static int
+Example_finalize(ExampleObject *self)
+{
+    Py_CLEAR(self->x_attr);
+    return 0;
+}
+
+static PyObject *
+Example_demo(ExampleObject *self, PyObject *args)
+{
+    PyObject *o = NULL;
+    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+        return NULL;
+    if (o != NULL && PyUnicode_Check(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyMethodDef Example_methods[] = {
+    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+        PyDoc_STR("demo() -> None")},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static PyObject *
+Example_getattro(ExampleObject *self, PyObject *name)
+{
+    if (self->x_attr != NULL) {
+        PyObject *v = PyDict_GetItem(self->x_attr, name);
+        if (v != NULL) {
+            Py_INCREF(v);
+            return v;
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)self, name);
+}
+
+static int
+Example_setattr(ExampleObject *self, char *name, PyObject *v)
+{
+    if (self->x_attr == NULL) {
+        self->x_attr = PyDict_New();
+        if (self->x_attr == NULL)
+            return -1;
+    }
+    if (v == NULL) {
+        int rv = PyDict_DelItemString(self->x_attr, name);
+        if (rv < 0)
+            PyErr_SetString(PyExc_AttributeError,
+                "delete non-existing Example attribute");
+        return rv;
+    }
+    else
+        return PyDict_SetItemString(self->x_attr, name, v);
+}
+
+static PyType_Slot Example_Type_slots[] = {
+    {Py_tp_doc, "The Example type"},
+//    {Py_tp_finalize, Example_finalize},
+    {Py_tp_traverse, Example_traverse},
+    {Py_tp_getattro, Example_getattro},
+    {Py_tp_setattr, Example_setattr},
+    {Py_tp_methods, Example_methods},
+    {0, 0},
+};
+
+static PyType_Spec Example_Type_spec = {
+    "_testimportexec.Example",
+    sizeof(ExampleObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+    Example_Type_slots
+};
+
+/* Function of two integers returning integer */
+
+PyDoc_STRVAR(testexport_foo_doc,
+"foo(i,j)\n\
+\n\
+Return the sum of i and j.");
+
+static PyObject *
+testexport_foo(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j))
+        return NULL;
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+/* Test that PyState registration fails  */
+
+//PyDoc_STRVAR(call_state_registration_func_doc,
+//"register_state(0): call PyState_FindModule()\n\
+//register_state(1): call PyState_AddModule()\n\
+//register_state(2): call PyState_RemoveModule()");
+//
+//static PyObject *
+//call_state_registration_func(PyObject *mod, PyObject *args)
+//{
+//    int i, ret;
+//    PyModuleDef *def = PyModule_GetDef(mod);
+//    if (def == NULL) {
+//        return NULL;
+//    }
+//    if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i))
+//        return NULL;
+//    switch (i) {
+//        case 0:
+//            mod = PyState_FindModule(def);
+//            if (mod == NULL) {
+//                Py_RETURN_NONE;
+//            }
+//            return mod;
+//        case 1:
+//            ret = PyState_AddModule(mod, def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//        case 2:
+//            ret = PyState_RemoveModule(def);
+//            if (ret != 0) {
+//                return NULL;
+//            }
+//            break;
+//    }
+//    Py_RETURN_NONE;
+//}
+
+
+static PyType_Slot Str_Type_slots[] = {
+    {Py_tp_base, NULL}, /* filled out in module exec function */
+    {0, 0},
+};
+
+static PyType_Spec Str_Type_spec = {
+    "_testimportexec.Str",
+    0,
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Str_Type_slots
+};
+
+static PyMethodDef testexport_methods[] = {
+    {"foo",             testexport_foo,         METH_VARARGS,
+        testexport_foo_doc},
+//    {"call_state_registration_func",  call_state_registration_func,
+//        METH_VARARGS, call_state_registration_func_doc},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static int execfunc(PyObject *m)
+{
+    PyObject *temp = NULL;
+
+    /* Due to cross platform compiler issues the slots must be filled
+     * here. It's required for portability to Windows without requiring
+     * C++. */
+    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+
+    /* Add a custom type */
+    temp = PyType_FromSpec(&Example_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Example", temp) != 0)
+        goto fail;
+
+    /* Add an exception type */
+    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "error", temp) != 0)
+        goto fail;
+
+    /* Add Str */
+    temp = PyType_FromSpec(&Str_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Str", temp) != 0)
+        goto fail;
+
+    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+        goto fail;
+
+    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+        goto fail;
+
+    return 0;
+ fail:
+    return -1;
+}
+
+/* Helper for module definitions; there'll be a lot of them */
+#define TEST_MODULE_DEF(name, slots, methods) { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */ \
+    name,                                       /* m_name */ \
+    PyDoc_STR("Test module " name),             /* m_doc */ \
+    0,                                          /* m_size */ \
+    methods,                                    /* m_methods */ \
+    slots,                                      /* m_slots */ \
+    NULL,                                       /* m_traverse */ \
+    NULL,                                       /* m_clear */ \
+    NULL,                                       /* m_free */ \
+}
+
+PyModuleDef_Slot main_slots[] = {
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef main_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+
+/**** Importing a non-module object ****/
+
+static PyModuleDef def_nonmodule;
+static PyModuleDef def_nonmodule_with_methods;
+
+/* Create a SimpleNamespace(three=3) */
+static PyObject*
+createfunc_nonmodule(PyObject *spec, PyModuleDef *def)
+{
+    PyObject *dct, *ns, *three;
+
+    if (def != &def_nonmodule && def != &def_nonmodule_with_methods) {
+        PyErr_SetString(PyExc_SystemError, "def does not match");
+        return NULL;
+    }
+
+    dct = PyDict_New();
+    if (dct == NULL)
+        return NULL;
+
+    three = PyLong_FromLong(3);
+    if (three == NULL) {
+        Py_DECREF(dct);
+        return NULL;
+    }
+    PyDict_SetItemString(dct, "three", three);
+    Py_DECREF(three);
+
+    ns = _PyNamespace_New(dct);
+    Py_DECREF(dct);
+    return ns;
+}
+
+static PyModuleDef_Slot slots_create_nonmodule[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule", slots_create_nonmodule, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule);
+}
+
+PyDoc_STRVAR(nonmodule_bar_doc,
+"bar(i,j)\n\
+\n\
+Return the difference of i - j.");
+
+static PyObject *
+nonmodule_bar(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:bar", &i, &j))
+        return NULL;
+    res = i - j;
+    return PyLong_FromLong(res);
+}
+
+static PyMethodDef nonmodule_methods[] = {
+    {"bar", nonmodule_bar, METH_VARARGS, nonmodule_bar_doc},
+    {NULL, NULL}           /* sentinel */
+};
+
+static PyModuleDef def_nonmodule_with_methods = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_methods", slots_create_nonmodule, nonmodule_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_methods(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_methods);
+}
+
+/**** Non-ASCII-named modules ****/
+
+static PyModuleDef def_nonascii_latin = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_latin",           /* m_name */
+    PyDoc_STR("Module named in Czech"),         /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU__testmultiphase_zkouka_naten_evc07gi8e(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_latin);
+}
+
+static PyModuleDef def_nonascii_kana = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_kana",            /* m_name */
+    PyDoc_STR("Module named in Japanese"),      /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_kana);
+}
+
+/*** Module with a single-character name ***/
+
+PyMODINIT_FUNC
+PyInit_x(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+/**** Testing NULL slots ****/
+
+static PyModuleDef null_slots_def = TEST_MODULE_DEF(
+    "_testmultiphase_null_slots", NULL, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_null_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&null_slots_def);
+}
+
+/**** Problematic modules ****/
+
+static PyModuleDef_Slot slots_bad_large[] = {
+    {_Py_mod_LAST_SLOT + 1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_large = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_large", slots_bad_large, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_large(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_large);
+}
+
+static PyModuleDef_Slot slots_bad_negative[] = {
+    {-1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_negative = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_negative", slots_bad_negative, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_negative(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_negative);
+}
+
+static PyModuleDef def_create_int_with_state = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "create_with_state",                        /* m_name */
+    PyDoc_STR("Not a PyModuleObject object, but requests per-module state"),
+    10,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_int_with_state(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_int_with_state);
+}
+
+
+static PyModuleDef def_negative_size = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "negative_size",                            /* m_name */
+    PyDoc_STR("PyModuleDef with negative m_size"),
+    -1,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_negative_size(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_negative_size);
+}
+
+
+static PyModuleDef uninitialized_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_uninitialized(PyObject *spec)
+{
+    return (PyObject*) &uninitialized_def;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_null(PyObject *spec)
+{
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_unreported_exception(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return PyModuleDef_Init(&main_def);
+}
+
+static PyObject*
+createfunc_null(PyObject *spec, PyModuleDef *def)
+{
+    return NULL;
+}
+
+PyModuleDef_Slot slots_create_null[] = {
+    {Py_mod_create, createfunc_null},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_null = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_null, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_null(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_null);
+}
+
+static PyObject*
+createfunc_raise(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return NULL;
+}
+
+static PyModuleDef_Slot slots_create_raise[] = {
+    {Py_mod_create, createfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_raise = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_raise(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_raise);
+}
+
+static PyObject*
+createfunc_unreported_exception(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return PyModule_New("foo");
+}
+
+static PyModuleDef_Slot slots_create_unreported_exception[] = {
+    {Py_mod_create, createfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_create_unreported_exception", slots_create_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_unreported_exception);
+}
+
+static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = {
+    {Py_mod_create, createfunc_nonmodule},
+    {Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef def_nonmodule_with_exec_slots = TEST_MODULE_DEF(
+    "_testmultiphase_nonmodule_with_exec_slots", slots_nonmodule_with_exec_slots, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_nonmodule_with_exec_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonmodule_with_exec_slots);
+}
+
+static int
+execfunc_err(PyObject *mod)
+{
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_err[] = {
+    {Py_mod_exec, execfunc_err},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_err = TEST_MODULE_DEF(
+    "_testmultiphase_exec_err", slots_exec_err, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_err(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_err);
+}
+
+static int
+execfunc_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_raise[] = {
+    {Py_mod_exec, execfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_raise = TEST_MODULE_DEF(
+    "_testmultiphase_exec_raise", slots_exec_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_raise(PyObject *mod)
+{
+    return PyModuleDef_Init(&def_exec_raise);
+}
+
+static int
+execfunc_unreported_exception(PyObject *mod)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return 0;
+}
+
+static PyModuleDef_Slot slots_exec_unreported_exception[] = {
+    {Py_mod_exec, execfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_exec_unreported_exception", slots_exec_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_unreported_exception);
+}
+
+/*** Helper for imp test ***/
+
+static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit_imp_dummy(PyObject *spec)
+{
+    return PyModuleDef_Init(&imp_dummy_def);
+}
diff --git a/lib_pypy/_testmultiphase.py b/lib_pypy/_testmultiphase.py
new file mode 100644
--- /dev/null
+++ b/lib_pypy/_testmultiphase.py
@@ -0,0 +1,18 @@
+import imp
+import os
+
+try:
+    import cpyext
+except ImportError:
+    raise ImportError("No module named '_testmultiphase'")
+import _pypy_testcapi
+cfile = '_testmultiphase.c'
+thisdir = os.path.dirname(__file__)
+output_dir = _pypy_testcapi.get_hashed_dir(os.path.join(thisdir, cfile))
+try:
+    fp, filename, description = imp.find_module('_test_multiphase', path=[output_dir])
+    with fp:
+        imp.load_module('_testmultiphase', fp, filename, description)
+except ImportError:
+    print('could not find _testmultiphase in %s' % output_dir)
+    _pypy_testcapi.compile_shared('_testmultiphase.c', '_testmultiphase', output_dir)
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -22,12 +22,27 @@
     linklibs = ['tcl', 'tk']
     libdirs = []
 else:
-    for _ver in ['', '8.6', '8.5', '']:
+    # On some Linux distributions, the tcl and tk libraries are
+    # stored in /usr/include, so we must check this case also
+    libdirs = []
+    found = False
+    for _ver in ['', '8.6', '8.5']:
         incdirs = ['/usr/include/tcl' + _ver]
         linklibs = ['tcl' + _ver, 'tk' + _ver]
-        libdirs = []
         if os.path.isdir(incdirs[0]):
+            found = True
             break
+    if not found:
+        for _ver in ['8.6', '8.5', '']:
+            incdirs = []
+            linklibs = ['tcl' + _ver, 'tk' + _ver]
+            if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
+                found = True
+                break
+    if not found:
+        sys.stderr.write("*** TCL libraries not found!  Falling back...\n")
+        incdirs = []
+        linklibs = ['tcl', 'tk']
 
 config_ffi = FFI()
 config_ffi.cdef("""
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -95,6 +95,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -1,7 +1,12 @@
 
 /***** Support code for embedding *****/
 
-#if defined(_MSC_VER)
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#if defined(_WIN32)
 #  define CFFI_DLLEXPORT  __declspec(dllexport)
 #elif defined(__GNUC__)
 #  define CFFI_DLLEXPORT  __attribute__((visibility("default")))
@@ -242,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -525,3 +530,7 @@
 #undef cffi_compare_and_swap
 #undef cffi_write_barrier
 #undef cffi_read_barrier
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -394,12 +394,17 @@
             replace_with = ' ' + replace_with
         return self._backend.getcname(cdecl, replace_with)
 
-    def gc(self, cdata, destructor):
+    def gc(self, cdata, destructor, size=0):
         """Return a new cdata object that points to the same
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
+
+        The optional 'size' gives an estimate of the size, used to
+        trigger the garbage collection more eagerly.  So far only used
+        on PyPy.  It tells the GC that the returned object keeps alive
+        roughly 'size' bytes of external memory.
         """
-        return self._backend.gcp(cdata, destructor)
+        return self._backend.gcp(cdata, destructor, size)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
 
     _weakref_cache_ref = None
 
-    def gcp(self, cdata, destructor):
+    def gcp(self, cdata, destructor, size=0):
         if self._weakref_cache_ref is None:
             import weakref
             class MyRef(weakref.ref):
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -412,6 +412,9 @@
             prnt('    }')
         prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
+        prnt('#if PY_MAJOR_VERSION >= 3')
+        prnt('    return NULL;')
+        prnt('#endif')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
         # 'export_symbols', so instead of fighting it, just give up and
@@ -578,7 +581,7 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.BasePrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -296,7 +296,7 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.PrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
@@ -872,6 +872,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -215,7 +215,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
diff --git a/lib_pypy/pyrepl/reader.py b/lib_pypy/pyrepl/reader.py
--- a/lib_pypy/pyrepl/reader.py
+++ b/lib_pypy/pyrepl/reader.py
@@ -239,6 +239,10 @@
 
     def __init__(self, console):
         self.buffer = []
+        # Enable the use of `insert` without a `prepare` call - necessary to
+        # facilitate the tab completion hack implemented for
+        # <https://bugs.python.org/issue25660>.
+        self.pos = 0
         self.ps1 = "->> "
         self.ps2 = "/>> "
         self.ps3 = "|.. "
diff --git a/lib_pypy/pyrepl/readline.py b/lib_pypy/pyrepl/readline.py
--- a/lib_pypy/pyrepl/readline.py
+++ b/lib_pypy/pyrepl/readline.py
@@ -297,10 +297,7 @@
         line = line.rstrip('\n')
         if isinstance(line, unicode):
             return line # on py3k
-        try:
-            return unicode(line, ENCODING)
-        except UnicodeDecodeError:   # bah, silently fall back...
-            return unicode(line, 'utf-8', 'replace')
+        return unicode(line, 'utf-8', 'replace')
 
     def get_history_length(self):
         return self.saved_history_length
@@ -317,7 +314,8 @@
         # history item: we use \r\n instead of just \n.  If the history
         # file is passed to GNU readline, the extra \r are just ignored.
         history = self.get_reader().history
-        f = open(os.path.expanduser(filename), 'r')
+        f = open(os.path.expanduser(filename), 'r', encoding='utf-8',
+                 errors='replace')
         buffer = []
         for line in f:
             if line.endswith('\r\n'):
@@ -334,15 +332,12 @@
     def write_history_file(self, filename='~/.history'):
         maxlength = self.saved_history_length
         history = self.get_reader().get_trimmed_history(maxlength)
-        f = open(os.path.expanduser(filename), 'w')
+        f = open(os.path.expanduser(filename), 'w', encoding='utf-8')
         for entry in history:
             # if we are on py3k, we don't need to encode strings before
             # writing it to a file
             if isinstance(entry, unicode) and sys.version_info < (3,):
-                try:
-                    entry = entry.encode(ENCODING)
-                except UnicodeEncodeError:   # bah, silently fall back...
-                    entry = entry.encode('utf-8')
+                entry = entry.encode('utf-8')
             entry = entry.replace('\n', '\r\n')   # multiline history support
             f.write(entry + '\n')
         f.close()
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -39,7 +39,7 @@
     "thread", "itertools", "pyexpat", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
-    "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
+    "_csv", "_pypyjson", "_posixsubprocess", # "_cppyy", "micronumpy"
     "_jitlog",
 ])
 
@@ -71,8 +71,12 @@
         if name in translation_modules:
             translation_modules.remove(name)
 
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # not tested on win32
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # not tested on win32
+    if "faulthandler" in working_modules:
+        working_modules.remove("faulthandler")  # missing details
+    if "_vmprof" in working_modules:
+        working_modules.remove("_vmprof")  # FIXME: missing details
 
     # The _locale module is needed by site.py on Windows
     default_modules.add("_locale")
@@ -81,8 +85,8 @@
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # depends on ctypes
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # depends on ctypes
 
 #if sys.platform.startswith("linux"):
 #    _mach = os.popen('uname -m', 'r').read().strip()
@@ -94,7 +98,7 @@
     '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
-    'cppyy': [('objspace.usemodules.cpyext', True)],
+    '_cppyy': [('objspace.usemodules.cpyext', True)],
     'faulthandler': [('objspace.usemodules._vmprof', True)],
     }

From pypy.commits at gmail.com  Thu Sep 28 16:36:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 28 Sep 2017 13:36:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix GCC warning caused by
 PySlice_GetIndicesEx()
Message-ID: <59cd5d4e.ceb1df0a.2a741.52b3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92493:8c01f25c3440
Date: 2017-09-28 22:35 +0200
http://bitbucket.org/pypy/pypy/changeset/8c01f25c3440/

Log:	Fix GCC warning caused by PySlice_GetIndicesEx()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1322,17 +1322,20 @@
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
     decls[pypy_decl].append("""
-        /* hack for https://bugs.python.org/issue29943 */
-        PyAPI_FUNC(int) %s(PySliceObject *arg0,
-                           Signed arg1, Signed *arg2,
-                           Signed *arg3, Signed *arg4, Signed *arg5);
-        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
-                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
-                Py_ssize_t *arg5) {
-            return %s(arg0, arg1, arg2, arg3,
-                      arg4, arg5);
-        }
-    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
+/* hack for https://bugs.python.org/issue29943 */
+
+PyAPI_FUNC(int) %s(PySliceObject *arg0,
+                    Signed arg1, Signed *arg2,
+                    Signed *arg3, Signed *arg4, Signed *arg5);
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+        Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+        Py_ssize_t *arg5) {
+    return %s(arg0, arg1, arg2, arg3,
+                arg4, arg5);
+}""" % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]

From pypy.commits at gmail.com  Thu Sep 28 17:00:36 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 28 Sep 2017 14:00:36 -0700 (PDT)
Subject: [pypy-commit] pypy default: Use C parser to define the
 PyXXXDescrObjects
Message-ID: <59cd62f4.1387df0a.4906a.0f85@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92494:8e1aad229f8e
Date: 2017-09-28 22:59 +0200
http://bitbucket.org/pypy/pypy/changeset/8e1aad229f8e/

Log:	Use C parser to define the PyXXXDescrObjects

diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,34 +1,6 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
 
-#define PyDescr_COMMON \
-    PyObject_HEAD \
-    PyTypeObject *d_type; \
-    PyObject *d_name
-
-typedef struct {
-    PyDescr_COMMON;
-} PyDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyMethodDef *d_method;
-} PyMethodDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct PyMemberDef *d_member;
-} PyMemberDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyGetSetDef *d_getset;
-} PyGetSetDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct wrapperbase *d_base;
-    void *d_wrapped; /* This can be any function pointer */
-} PyWrapperDescrObject;
+#include "cpyext_descrobject.h"
 
 #endif
diff --git a/pypy/module/cpyext/parse/cpyext_descrobject.h b/pypy/module/cpyext/parse/cpyext_descrobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_descrobject.h
@@ -0,0 +1,29 @@
+#define PyDescr_COMMON \
+    PyObject_HEAD \
+    PyTypeObject *d_type; \
+    PyObject *d_name
+
+typedef struct {
+    PyDescr_COMMON;
+} PyDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyMethodDef *d_method;
+} PyMethodDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct PyMemberDef *d_member;
+} PyMemberDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyGetSetDef *d_getset;
+} PyGetSetDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct wrapperbase *d_base;
+    void *d_wrapped; /* This can be any function pointer */
+} PyWrapperDescrObject;
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -23,7 +23,6 @@
     Py_TPFLAGS_TYPE_SUBCLASS,
     Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
     )
-from pypy.module.cpyext.cparser import parse_source
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -42,6 +41,8 @@
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
+cts.parse_header(parse_dir / "cpyext_descrobject.h")
+
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
 PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
@@ -116,57 +117,24 @@
     )
 assert not W_MemberDescr.typedef.acceptable_as_base_class  # no __new__
 
-PyDescrObject = lltype.ForwardReference()
-PyDescrObjectPtr = lltype.Ptr(PyDescrObject)
-PyDescrObjectFields = PyObjectFields + (
-    ("d_type", PyTypeObjectPtr),
-    ("d_name", PyObject),
-    )
-cpython_struct("PyDescrObject", PyDescrObjectFields,
-               PyDescrObject)
-
-PyMemberDescrObjectStruct = lltype.ForwardReference()
-PyMemberDescrObject = lltype.Ptr(PyMemberDescrObjectStruct)
-PyMemberDescrObjectFields = PyDescrObjectFields + (
-    ("d_member", lltype.Ptr(PyMemberDef)),
-    )
-cpython_struct("PyMemberDescrObject", PyMemberDescrObjectFields,
-               PyMemberDescrObjectStruct, level=2)
-
-PyGetSetDescrObjectStruct = lltype.ForwardReference()
-PyGetSetDescrObject = lltype.Ptr(PyGetSetDescrObjectStruct)
-PyGetSetDescrObjectFields = PyDescrObjectFields + (
-    ("d_getset", lltype.Ptr(PyGetSetDef)),
-    )
-cpython_struct("PyGetSetDescrObject", PyGetSetDescrObjectFields,
-               PyGetSetDescrObjectStruct, level=2)
-
-PyMethodDescrObjectStruct = lltype.ForwardReference()
-PyMethodDescrObject = lltype.Ptr(PyMethodDescrObjectStruct)
-PyMethodDescrObjectFields = PyDescrObjectFields + (
-    ("d_method", lltype.Ptr(PyMethodDef)),
-    )
-cpython_struct("PyMethodDescrObject", PyMethodDescrObjectFields,
-               PyMethodDescrObjectStruct, level=2)
-
 @bootstrap_function
 def init_memberdescrobject(space):
     make_typedescr(W_MemberDescr.typedef,
-                   basestruct=PyMemberDescrObject.TO,
+                   basestruct=cts.gettype('PyMemberDescrObject'),
                    attach=memberdescr_attach,
                    realize=memberdescr_realize,
                    )
     make_typedescr(W_GetSetPropertyEx.typedef,
-                   basestruct=PyGetSetDescrObject.TO,
+                   basestruct=cts.gettype('PyGetSetDescrObject'),
                    attach=getsetdescr_attach,
                    )
     make_typedescr(W_PyCClassMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=classmethoddescr_realize,
                    )
     make_typedescr(W_PyCMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=methoddescr_realize,
                    )
@@ -176,7 +144,7 @@
     Fills a newly allocated PyMemberDescrObject with the given W_MemberDescr
     object. The values must not be modified.
     """
-    py_memberdescr = rffi.cast(PyMemberDescrObject, py_obj)
+    py_memberdescr = cts.cast('PyMemberDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_MemberDescr)
     py_memberdescr.c_d_member = w_obj.member
@@ -195,7 +163,7 @@
     Fills a newly allocated PyGetSetDescrObject with the given W_GetSetPropertyEx
     object. The values must not be modified.
     """
-    py_getsetdescr = rffi.cast(PyGetSetDescrObject, py_obj)
+    py_getsetdescr = cts.cast('PyGetSetDescrObject*', py_obj)
     if isinstance(w_obj, GetSetProperty):
         py_getsetdef = make_GetSet(space, w_obj)
         assert space.isinstance_w(w_userdata, space.w_type)
@@ -207,7 +175,7 @@
     py_getsetdescr.c_d_getset = w_obj.getset
 
 def methoddescr_attach(space, py_obj, w_obj, w_userdata=None):
-    py_methoddescr = rffi.cast(PyMethodDescrObject, py_obj)
+    py_methoddescr = cts.cast('PyMethodDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_PyCFunctionObject)
     py_methoddescr.c_d_method = w_obj.ml
@@ -452,7 +420,7 @@
         pto.c_tp_flags |= Py_TPFLAGS_INT_SUBCLASS
     elif space.issubtype_w(w_obj, space.w_long):
         pto.c_tp_flags |= Py_TPFLAGS_LONG_SUBCLASS
-    elif space.issubtype_w(w_obj, space.w_bytes): 
+    elif space.issubtype_w(w_obj, space.w_bytes):
         pto.c_tp_flags |= Py_TPFLAGS_STRING_SUBCLASS # STRING->BYTES on py3
     elif space.issubtype_w(w_obj, space.w_unicode):
         pto.c_tp_flags |= Py_TPFLAGS_UNICODE_SUBCLASS

From pypy.commits at gmail.com  Thu Sep 28 17:49:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 28 Sep 2017 14:49:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59cd6e5d.0ec61c0a.ca3e0.816d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92495:9d4f0cdad713
Date: 2017-09-28 23:48 +0200
http://bitbucket.org/pypy/pypy/changeset/9d4f0cdad713/

Log:	hg merge default

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1338,17 +1338,20 @@
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
     decls[pypy_decl].append("""
-        /* hack for https://bugs.python.org/issue29943 */
-        PyAPI_FUNC(int) %s(PySliceObject *arg0,
-                           Signed arg1, Signed *arg2,
-                           Signed *arg3, Signed *arg4, Signed *arg5);
-        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
-                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
-                Py_ssize_t *arg5) {
-            return %s(arg0, arg1, arg2, arg3,
-                      arg4, arg5);
-        }
-    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
+/* hack for https://bugs.python.org/issue29943 */
+
+PyAPI_FUNC(int) %s(PySliceObject *arg0,
+                    Signed arg1, Signed *arg2,
+                    Signed *arg3, Signed *arg4, Signed *arg5);
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+        Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+        Py_ssize_t *arg5) {
+    return %s(arg0, arg1, arg2, arg3,
+                arg4, arg5);
+}""" % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]
diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,34 +1,6 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
 
-#define PyDescr_COMMON \
-    PyObject_HEAD \
-    PyTypeObject *d_type; \
-    PyObject *d_name
-
-typedef struct {
-    PyDescr_COMMON;
-} PyDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyMethodDef *d_method;
-} PyMethodDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct PyMemberDef *d_member;
-} PyMemberDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyGetSetDef *d_getset;
-} PyGetSetDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct wrapperbase *d_base;
-    void *d_wrapped; /* This can be any function pointer */
-} PyWrapperDescrObject;
+#include "cpyext_descrobject.h"
 
 #endif
diff --git a/pypy/module/cpyext/parse/cpyext_descrobject.h b/pypy/module/cpyext/parse/cpyext_descrobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_descrobject.h
@@ -0,0 +1,29 @@
+#define PyDescr_COMMON \
+    PyObject_HEAD \
+    PyTypeObject *d_type; \
+    PyObject *d_name
+
+typedef struct {
+    PyDescr_COMMON;
+} PyDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyMethodDef *d_method;
+} PyMethodDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct PyMemberDef *d_member;
+} PyMemberDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyGetSetDef *d_getset;
+} PyGetSetDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct wrapperbase *d_base;
+    void *d_wrapped; /* This can be any function pointer */
+} PyWrapperDescrObject;
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -19,7 +19,8 @@
     Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
     Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
     Py_TPFLAGS_TYPE_SUBCLASS,
-    Py_TPFLAGS_BYTES_SUBCLASS)
+    Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
+    )
 from pypy.module.cpyext.cparser import CTypeSpace
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -39,6 +40,8 @@
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
+cts.parse_header(parse_dir / "cpyext_descrobject.h")
+
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
 PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
@@ -115,57 +118,24 @@
     )
 assert not W_MemberDescr.typedef.acceptable_as_base_class  # no __new__
 
-PyDescrObject = lltype.ForwardReference()
-PyDescrObjectPtr = lltype.Ptr(PyDescrObject)
-PyDescrObjectFields = PyObjectFields + (
-    ("d_type", PyTypeObjectPtr),
-    ("d_name", PyObject),
-    )
-cpython_struct("PyDescrObject", PyDescrObjectFields,
-               PyDescrObject)
-
-PyMemberDescrObjectStruct = lltype.ForwardReference()
-PyMemberDescrObject = lltype.Ptr(PyMemberDescrObjectStruct)
-PyMemberDescrObjectFields = PyDescrObjectFields + (
-    ("d_member", lltype.Ptr(PyMemberDef)),
-    )
-cpython_struct("PyMemberDescrObject", PyMemberDescrObjectFields,
-               PyMemberDescrObjectStruct, level=2)
-
-PyGetSetDescrObjectStruct = lltype.ForwardReference()
-PyGetSetDescrObject = lltype.Ptr(PyGetSetDescrObjectStruct)
-PyGetSetDescrObjectFields = PyDescrObjectFields + (
-    ("d_getset", lltype.Ptr(PyGetSetDef)),
-    )
-cpython_struct("PyGetSetDescrObject", PyGetSetDescrObjectFields,
-               PyGetSetDescrObjectStruct, level=2)
-
-PyMethodDescrObjectStruct = lltype.ForwardReference()
-PyMethodDescrObject = lltype.Ptr(PyMethodDescrObjectStruct)
-PyMethodDescrObjectFields = PyDescrObjectFields + (
-    ("d_method", lltype.Ptr(PyMethodDef)),
-    )
-cpython_struct("PyMethodDescrObject", PyMethodDescrObjectFields,
-               PyMethodDescrObjectStruct, level=2)
-
 @bootstrap_function
 def init_memberdescrobject(space):
     make_typedescr(W_MemberDescr.typedef,
-                   basestruct=PyMemberDescrObject.TO,
+                   basestruct=cts.gettype('PyMemberDescrObject'),
                    attach=memberdescr_attach,
                    realize=memberdescr_realize,
                    )
     make_typedescr(W_GetSetPropertyEx.typedef,
-                   basestruct=PyGetSetDescrObject.TO,
+                   basestruct=cts.gettype('PyGetSetDescrObject'),
                    attach=getsetdescr_attach,
                    )
     make_typedescr(W_PyCClassMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=classmethoddescr_realize,
                    )
     make_typedescr(W_PyCMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=methoddescr_realize,
                    )
@@ -175,7 +145,7 @@
     Fills a newly allocated PyMemberDescrObject with the given W_MemberDescr
     object. The values must not be modified.
     """
-    py_memberdescr = rffi.cast(PyMemberDescrObject, py_obj)
+    py_memberdescr = cts.cast('PyMemberDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_MemberDescr)
     py_memberdescr.c_d_member = w_obj.member
@@ -194,7 +164,7 @@
     Fills a newly allocated PyGetSetDescrObject with the given W_GetSetPropertyEx
     object. The values must not be modified.
     """
-    py_getsetdescr = rffi.cast(PyGetSetDescrObject, py_obj)
+    py_getsetdescr = cts.cast('PyGetSetDescrObject*', py_obj)
     if isinstance(w_obj, GetSetProperty):
         py_getsetdef = make_GetSet(space, w_obj)
         assert space.isinstance_w(w_userdata, space.w_type)
@@ -206,7 +176,7 @@
     py_getsetdescr.c_d_getset = w_obj.getset
 
 def methoddescr_attach(space, py_obj, w_obj, w_userdata=None):
-    py_methoddescr = rffi.cast(PyMethodDescrObject, py_obj)
+    py_methoddescr = cts.cast('PyMethodDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_PyCFunctionObject)
     py_methoddescr.c_d_method = w_obj.ml

From pypy.commits at gmail.com  Thu Sep 28 18:19:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 28 Sep 2017 15:19:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix merge
Message-ID: <59cd7576.8b141c0a.57a7e.68b3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92496:8b44bf979649
Date: 2017-09-29 00:18 +0200
http://bitbucket.org/pypy/pypy/changeset/8b44bf979649/

Log:	fix merge

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -1340,13 +1340,13 @@
     decls[pypy_decl].append("""
 /* hack for https://bugs.python.org/issue29943 */
 
-PyAPI_FUNC(int) %s(PySliceObject *arg0,
+PyAPI_FUNC(int) %s(PyObject *arg0,
                     Signed arg1, Signed *arg2,
                     Signed *arg3, Signed *arg4, Signed *arg5);
 #ifdef __GNUC__
 __attribute__((__unused__))
 #endif
-static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
+static int PySlice_GetIndicesEx(PyObject *arg0, Py_ssize_t arg1,
         Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
         Py_ssize_t *arg5) {
     return %s(arg0, arg1, arg2, arg3,
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -12,15 +12,14 @@
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t,
     slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
+    build_type_checkers_flags, cts, parse_dir, PyTypeObject,
     PyTypeObjectPtr, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
     Py_TPFLAGS_TUPLE_SUBCLASS, Py_TPFLAGS_UNICODE_SUBCLASS,
     Py_TPFLAGS_DICT_SUBCLASS, Py_TPFLAGS_BASE_EXC_SUBCLASS,
     Py_TPFLAGS_TYPE_SUBCLASS,
-    Py_TPFLAGS_INT_SUBCLASS, Py_TPFLAGS_STRING_SUBCLASS, # change on py3
-    )
+    Py_TPFLAGS_BYTES_SUBCLASS)
 from pypy.module.cpyext.cparser import CTypeSpace
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
@@ -40,7 +39,6 @@
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
-cts.parse_header(parse_dir / "cpyext_descrobject.h")
 
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
@@ -48,6 +46,7 @@
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
+cts.parse_header(parse_dir / "cpyext_descrobject.h")
 cts.parse_header(parse_dir / "typeslots.h")
 
 

From pypy.commits at gmail.com  Thu Sep 28 18:35:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 28 Sep 2017 15:35:24 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Match CPython3.5's descrobject.h
Message-ID: <59cd792c.0988df0a.3ffcd.5afd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92497:c88a0030175d
Date: 2017-09-29 00:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c88a0030175d/

Log:	Match CPython3.5's descrobject.h

diff --git a/pypy/module/cpyext/parse/cpyext_descrobject.h b/pypy/module/cpyext/parse/cpyext_descrobject.h
--- a/pypy/module/cpyext/parse/cpyext_descrobject.h
+++ b/pypy/module/cpyext/parse/cpyext_descrobject.h
@@ -1,11 +1,11 @@
-#define PyDescr_COMMON \
-    PyObject_HEAD \
-    PyTypeObject *d_type; \
-    PyObject *d_name
+typedef struct {
+    PyObject_HEAD
+    PyTypeObject *d_type;
+    PyObject *d_name;
+    PyObject *d_qualname;
+} PyDescrObject;
 
-typedef struct {
-    PyDescr_COMMON;
-} PyDescrObject;
+#define PyDescr_COMMON PyDescrObject d_common
 
 typedef struct {
     PyDescr_COMMON;

From pypy.commits at gmail.com  Fri Sep 29 04:38:14 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 29 Sep 2017 01:38:14 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: help testrunner find pypy3 exe on win32
Message-ID: <59ce0676.c1acdf0a.90297.9fce@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r92498:48bda039069f
Date: 2017-09-29 11:37 +0300
http://bitbucket.org/pypy/pypy/changeset/48bda039069f/

Log:	help testrunner find pypy3 exe on win32

diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg
--- a/pypy/pytest-A.cfg
+++ b/pypy/pytest-A.cfg
@@ -1,5 +1,8 @@
+import sys
 cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module']
 
 interp = ['python']
-test_driver = ['test_all.py', '-A', '--python=goal/pypy3-c']
-
+if sys.platform == 'win32':
+    test_driver = ['test_all.py', '-A', '--python=goal/pypy3-cw.exe']
+else:
+    test_driver = ['test_all.py', '-A', '--python=goal/pypy3-c']

From pypy.commits at gmail.com  Fri Sep 29 05:38:23 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 02:38:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <59ce148f.1d8bdf0a.e770e.a4bc@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92499:2fef598a1c41
Date: 2017-09-29 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/2fef598a1c41/

Log:	fix test

diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test/test_fileobject.py
--- a/pypy/module/cpyext/test/test_fileobject.py
+++ b/pypy/module/cpyext/test/test_fileobject.py
@@ -7,7 +7,7 @@
         module = self.import_extension('foo', [
             ("defenc", "METH_NOARGS",
              """
-                return PyString_FromString(Py_FileSystemDefaultEncoding);
+                return PyUnicode_FromString(Py_FileSystemDefaultEncoding);
              """),
             ])
         assert module.defenc() == sys.getfilesystemencoding()

From pypy.commits at gmail.com  Fri Sep 29 06:17:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 03:17:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix Cython issue with classmethods in
 cdef classes
Message-ID: <59ce1dd1.0ec61c0a.ca3e0.cee7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92500:3c8212fb97a5
Date: 2017-09-29 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/3c8212fb97a5/

Log:	fix Cython issue with classmethods in cdef classes

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -669,7 +669,7 @@
         'PySlice_Type': 'space.gettypeobject(W_SliceObject.typedef)',
         'PyStaticMethod_Type': 'space.gettypeobject(StaticMethod.typedef)',
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
-        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
+        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)',
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -10,8 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL,
-    PyTypeObjectPtr, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -102,7 +101,7 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        w_objclass = self.w_objclass 
+        w_objclass = self.w_objclass
         assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
             self.name, w_objclass.name))
@@ -110,7 +109,7 @@
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%8' of '%s' object needs an argument",
@@ -118,7 +117,7 @@
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%8' requires a '%s' object but received a '%T'",
@@ -323,11 +322,15 @@
 def PyClassMethod_New(space, w_func):
     return ClassMethod(w_func)
 
- at cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewMethod(space, w_type, method):
     return W_PyCMethodObject(space, method, w_type)
 
- at cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewClassMethod(space, w_type, method):
     return W_PyCClassMethodObject(space, method, w_type)
 
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,6 +83,22 @@
     return cls;
 }
 
+PyObject* make_classmethod(PyObject* method)
+{
+    // adapted from __Pyx_Method_ClassMethod
+    if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) {
+        return PyClassMethod_New(method);
+    }
+    else if (PyMethod_Check(method)) {
+        return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
+    }
+    else {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_common.d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
+    }
+}
+
 static PyObject *
 foo_unset(fooobject *self)
 {
@@ -95,6 +111,7 @@
     {"copy",      (PyCFunction)foo_copy,      METH_NOARGS,  NULL},
     {"create",    (PyCFunction)foo_create,    METH_NOARGS|METH_STATIC,  NULL},
     {"classmeth", (PyCFunction)foo_classmeth, METH_NOARGS|METH_CLASS,  NULL},
+    {"fake_classmeth", (PyCFunction)foo_classmeth, METH_NOARGS,  NULL},
     {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL},
     {NULL, NULL}                 /* sentinel */
 };
@@ -167,19 +184,19 @@
     /* copied from numpy scalartypes.c for inherited classes */
     if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
     {
-        PyTypeObject *sup; 
-        /* We are inheriting from a Python type as well so 
+        PyTypeObject *sup;
+        /* We are inheriting from a Python type as well so
            give it first dibs on conversion */
         sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
-        /* Prevent recursion */ 
-        if (new_fooType != sup->tp_new) 
+        /* Prevent recursion */
+        if (new_fooType != sup->tp_new)
         {
             o = sup->tp_new(t, args, kwds);
             return o;
         }
     }
     o = t->tp_alloc(t, 0);
-    return o;   
+    return o;
 };
 
 static PyMemberDef foo_members[] = {
@@ -717,7 +734,7 @@
     "foo",
     "Module Doc",
     -1,
-    foo_functions, 
+    foo_functions,
     NULL,
     NULL,
     NULL,
@@ -751,6 +768,7 @@
 #endif
 {
     PyObject *d;
+    PyObject *fake_classmeth, *classmeth;
 #if PY_MAJOR_VERSION >= 3
     PyObject *module = PyModule_Create(&moduledef);
 #else
@@ -808,6 +826,10 @@
         INITERROR;
     gettype2 = PyObject_New(PyObject, &GetType2);
 
+    fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
+    classmeth = make_classmethod(fake_classmeth);
+    if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
+        INITERROR;
 
     d = PyModule_GetDict(module);
     if (d == NULL)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -135,6 +135,12 @@
             "<method 'copy' of 'foo' objects>")
         raises(TypeError, descr, None)
 
+    def test_cython_fake_classmethod(self):
+        module = self.import_module(name='foo')
+        print(module.fooType.fake_classmeth)
+        print(type(module.fooType.fake_classmeth))
+        assert module.fooType.fake_classmeth() is module.fooType
+
     def test_new(self):
         # XXX cpython segfaults but if run singly (with -k test_new) this passes
         module = self.import_module(name='foo')

From pypy.commits at gmail.com  Fri Sep 29 07:01:44 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 29 Sep 2017 04:01:44 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: (arigo,
 fijal) add accounting for basic memory pressure done by raw mallocs
Message-ID: <59ce2818.d5c21c0a.2092d.5673@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92501:f0e56ca3a8d6
Date: 2017-09-29 13:01 +0200
http://bitbucket.org/pypy/pypy/changeset/f0e56ca3a8d6/

Log:	(arigo, fijal) add accounting for basic memory pressure done by raw
	mallocs

diff --git a/rpython/memory/gc/base.py b/rpython/memory/gc/base.py
--- a/rpython/memory/gc/base.py
+++ b/rpython/memory/gc/base.py
@@ -83,7 +83,9 @@
                             has_custom_trace,
                             fast_path_tracing,
                             has_gcptr,
-                            cannot_pin):
+                            cannot_pin,
+                            has_memory_pressure,
+                            get_memory_pressure_ofs):
         self.finalizer_handlers = finalizer_handlers
         self.destructor_or_custom_trace = destructor_or_custom_trace
         self.is_old_style_finalizer = is_old_style_finalizer
@@ -103,6 +105,8 @@
         self.fast_path_tracing = fast_path_tracing
         self.has_gcptr = has_gcptr
         self.cannot_pin = cannot_pin
+        self.has_memory_pressure = has_memory_pressure
+        self.get_memory_pressure_ofs = get_memory_pressure_ofs
 
     def get_member_index(self, type_id):
         return self.member_index(type_id)
diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -2920,6 +2920,12 @@
         self.old_objects_with_weakrefs = new_with_weakref
 
     def get_stats(self, stats_no):
+        from rpython.memory.gc import inspector
+
+        if stats_no == rgc.TOTAL_MEMORY:
+            return 0
+        elif stats_no == rgc.TOTAL_MEMORY_PRESSURE:
+            return inspector.count_memory_pressure(self)
         return 0
 
 
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -92,17 +92,12 @@
 
 AddressStack = get_address_stack()
 
-class HeapDumper(object):
-    _alloc_flavor_ = "raw"
-    BUFSIZE = 8192     # words
+class BaseWalker(object):
+    _alloc_flavor_ = 'raw'
 
-    def __init__(self, gc, fd):
+    def __init__(self, gc):
         self.gc = gc
         self.gcflag = gc.gcflag_extra
-        self.fd = rffi.cast(rffi.INT, fd)
-        self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE,
-                                         flavor='raw')
-        self.buf_count = 0
         if self.gcflag == 0:
             self.seen = AddressDict()
         self.pending = AddressStack()
@@ -111,8 +106,102 @@
         if self.gcflag == 0:
             self.seen.delete()
         self.pending.delete()
+        free_non_gc_object(self)
+
+    def add_roots(self):
+        self.gc.enumerate_all_roots(_hd_add_root, self)
+        pendingroots = self.pending
+        self.pending = AddressStack()
+        self.walk(pendingroots)
+        pendingroots.delete()
+        self.end_add_roots_marker()
+
+    def end_add_roots_marker(self):
+        pass
+
+    def add(self, obj):
+        if self.gcflag == 0:
+            if not self.seen.contains(obj):
+                self.seen.setitem(obj, obj)
+                self.pending.append(obj)
+        else:
+            hdr = self.gc.header(obj)
+            if (hdr.tid & self.gcflag) == 0:
+                hdr.tid |= self.gcflag
+                self.pending.append(obj)
+
+    def walk(self, pending):
+        while pending.non_empty():
+            self.processobj(pending.pop())
+
+    # ----------
+    # A simplified copy of the above, to make sure we walk again all the
+    # objects to clear the 'gcflag'.
+
+    def unobj(self, obj):
+        gc = self.gc
+        gc.trace(obj, self._unref, None)
+
+    def _unref(self, pointer, _):
+        obj = pointer.address[0]
+        self.unadd(obj)
+
+    def unadd(self, obj):
+        assert self.gcflag != 0
+        hdr = self.gc.header(obj)
+        if (hdr.tid & self.gcflag) != 0:
+            hdr.tid &= ~self.gcflag
+            self.pending.append(obj)
+
+    def clear_gcflag_again(self):
+        self.gc.enumerate_all_roots(_hd_unadd_root, self)
+        pendingroots = self.pending
+        self.pending = AddressStack()
+        self.unwalk(pendingroots)
+        pendingroots.delete()
+
+    def unwalk(self, pending):
+        while pending.non_empty():
+            self.unobj(pending.pop())
+
+    def finish_processing(self):
+        if self.gcflag != 0:
+            self.clear_gcflag_again()
+            self.unwalk(self.pending)
+
+    def process(self):
+        self.add_roots()
+        self.walk(self.pending)
+
+
+class MemoryPressureCounter(BaseWalker):
+
+    def __init__(self, gc):
+        self.count = 0
+        BaseWalker.__init__(self, gc)
+
+    def processobj(self, obj):
+        gc = self.gc
+        typeid = gc.get_type_id(obj)
+        if gc.has_memory_pressure(typeid):
+            ofs = gc.get_memory_pressure_ofs(typeid)
+            val = (obj + ofs).signed[0]
+            self.count += val
+
+
+class HeapDumper(BaseWalker):
+    BUFSIZE = 8192     # words
+
+    def __init__(self, gc, fd):
+        BaseWalker.__init__(self, gc)
+        self.fd = rffi.cast(rffi.INT, fd)
+        self.writebuffer = lltype.malloc(rffi.SIGNEDP.TO, self.BUFSIZE,
+                                         flavor='raw')
+        self.buf_count = 0
+
+    def delete(self):
         lltype.free(self.writebuffer, flavor='raw')
-        free_non_gc_object(self)
+        BaseWalker.delete(self)
 
     @jit.dont_look_inside
     def flush(self):
@@ -143,6 +232,7 @@
         self.write(0)
         self.write(0)
         self.write(-1)
+    end_add_roots_marker = write_marker
 
     def writeobj(self, obj):
         gc = self.gc
@@ -152,64 +242,13 @@
         self.write(gc.get_size_incl_hash(obj))
         gc.trace(obj, self._writeref, None)
         self.write(-1)
+    processobj = writeobj
 
     def _writeref(self, pointer, _):
         obj = pointer.address[0]
         self.write(llmemory.cast_adr_to_int(obj))
         self.add(obj)
 
-    def add(self, obj):
-        if self.gcflag == 0:
-            if not self.seen.contains(obj):
-                self.seen.setitem(obj, obj)
-                self.pending.append(obj)
-        else:
-            hdr = self.gc.header(obj)
-            if (hdr.tid & self.gcflag) == 0:
-                hdr.tid |= self.gcflag
-                self.pending.append(obj)
-
-    def add_roots(self):
-        self.gc.enumerate_all_roots(_hd_add_root, self)
-        pendingroots = self.pending
-        self.pending = AddressStack()
-        self.walk(pendingroots)
-        pendingroots.delete()
-        self.write_marker()
-
-    def walk(self, pending):
-        while pending.non_empty():
-            self.writeobj(pending.pop())
-
-    # ----------
-    # A simplified copy of the above, to make sure we walk again all the
-    # objects to clear the 'gcflag'.
-
-    def unwriteobj(self, obj):
-        gc = self.gc
-        gc.trace(obj, self._unwriteref, None)
-
-    def _unwriteref(self, pointer, _):
-        obj = pointer.address[0]
-        self.unadd(obj)
-
-    def unadd(self, obj):
-        assert self.gcflag != 0
-        hdr = self.gc.header(obj)
-        if (hdr.tid & self.gcflag) != 0:
-            hdr.tid &= ~self.gcflag
-            self.pending.append(obj)
-
-    def clear_gcflag_again(self):
-        self.gc.enumerate_all_roots(_hd_unadd_root, self)
-        pendingroots = self.pending
-        self.pending = AddressStack()
-        self.unwalk(pendingroots)
-        pendingroots.delete()
-
-    def unwalk(self, pending):
-        while pending.non_empty():
-            self.unwriteobj(pending.pop())
 
 def _hd_add_root(obj, heap_dumper):
     heap_dumper.add(obj)
@@ -219,15 +258,20 @@
 
 def dump_rpy_heap(gc, fd):
     heapdumper = HeapDumper(gc, fd)
-    heapdumper.add_roots()
-    heapdumper.walk(heapdumper.pending)
+    heapdumper.process()
     heapdumper.flush()
-    if heapdumper.gcflag != 0:
-        heapdumper.clear_gcflag_again()
-        heapdumper.unwalk(heapdumper.pending)
+    heapdumper.finish_processing()
     heapdumper.delete()
     return True
 
+def count_memory_pressure(gc):
+    counter = MemoryPressureCounter(gc)
+    counter.process()
+    counter.finish_processing()
+    res = counter.count
+    counter.delete()
+    return res
+
 def get_typeids_z(gc):
     srcaddress = gc.root_walker.gcdata.typeids_z
     return llmemory.cast_adr_to_ptr(srcaddress, lltype.Ptr(rgc.ARRAY_OF_CHAR))
diff --git a/rpython/memory/gctypelayout.py b/rpython/memory/gctypelayout.py
--- a/rpython/memory/gctypelayout.py
+++ b/rpython/memory/gctypelayout.py
@@ -21,13 +21,21 @@
     # A destructor is called when the object is about to be freed.
     # A custom tracer (CT) enumerates the addresses that contain GCREFs.
     # Both are called with the address of the object as only argument.
+    # They're embedded in a struct that has raw_memory_offset as another
+    # argument, which is only valid if T_HAS_MEMORY_PRESSURE is set
     CUSTOM_FUNC = lltype.FuncType([llmemory.Address], lltype.Void)
     CUSTOM_FUNC_PTR = lltype.Ptr(CUSTOM_FUNC)
+    CUSTOM_DATA_STRUCT = lltype.Struct('custom_data',
+        ('customfunc', CUSTOM_FUNC_PTR),
+        ('memory_pressure_offset', lltype.Signed), # offset to where the amount
+                                           # of owned memory pressure is stored
+        )
+    CUSTOM_DATA_STRUCT_PTR = lltype.Ptr(CUSTOM_DATA_STRUCT)
 
     # structure describing the layout of a typeid
     TYPE_INFO = lltype.Struct("type_info",
         ("infobits",       lltype.Signed),    # combination of the T_xxx consts
-        ("customfunc",     CUSTOM_FUNC_PTR),
+        ("customdata",     CUSTOM_DATA_STRUCT_PTR),
         ("fixedsize",      lltype.Signed),
         ("ofstoptrs",      lltype.Ptr(OFFSETS_TO_GC_PTR)),
         hints={'immutable': True},
@@ -81,14 +89,14 @@
     def q_cannot_pin(self, typeid):
         typeinfo = self.get(typeid)
         ANY = (T_HAS_GCPTR | T_IS_WEAKREF)
-        return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customfunc)
+        return (typeinfo.infobits & ANY) != 0 or bool(typeinfo.customdata)
 
     def q_finalizer_handlers(self):
         adr = self.finalizer_handlers   # set from framework.py or gcwrapper.py
         return llmemory.cast_adr_to_ptr(adr, lltype.Ptr(FIN_HANDLER_ARRAY))
 
     def q_destructor_or_custom_trace(self, typeid):
-        return self.get(typeid).customfunc
+        return self.get(typeid).customdata.customfunc
 
     def q_is_old_style_finalizer(self, typeid):
         typeinfo = self.get(typeid)
@@ -139,6 +147,15 @@
         infobits = self.get(typeid).infobits
         return infobits & T_ANY_SLOW_FLAG == 0
 
+    def q_has_memory_pressure(self, typeid):
+        infobits = self.get(typeid).infobits
+        return infobits & T_HAS_MEMORY_PRESSURE != 0
+
+    def q_get_memory_pressure_ofs(self, typeid):
+        infobits = self.get(typeid).infobits
+        assert infobits & T_HAS_MEMORY_PRESSURE != 0
+        return self.get(typeid).customdata.memory_pressure_offset
+
     def set_query_functions(self, gc):
         gc.set_query_functions(
             self.q_is_varsize,
@@ -159,7 +176,9 @@
             self.q_has_custom_trace,
             self.q_fast_path_tracing,
             self.q_has_gcptr,
-            self.q_cannot_pin)
+            self.q_cannot_pin,
+            self.q_has_memory_pressure,
+            self.q_get_memory_pressure_ofs)
 
     def _has_got_custom_trace(self, typeid):
         type_info = self.get(typeid)
@@ -176,8 +195,9 @@
 T_HAS_CUSTOM_TRACE          = 0x200000
 T_HAS_OLDSTYLE_FINALIZER    = 0x400000
 T_HAS_GCPTR                 = 0x1000000
-T_KEY_MASK                  = intmask(0xFE000000) # bug detection only
-T_KEY_VALUE                 = intmask(0x5A000000) # bug detection only
+T_HAS_MEMORY_PRESSURE       = 0x2000000 # first field is memory pressure field
+T_KEY_MASK                  = intmask(0xFC000000) # bug detection only
+T_KEY_VALUE                 = intmask(0x58000000) # bug detection only
 
 def _check_valid_type_info(p):
     ll_assert(p.infobits & T_KEY_MASK == T_KEY_VALUE, "invalid type_id")
@@ -192,6 +212,25 @@
     ll_assert(llop.is_group_member_nonzero(lltype.Bool, typeid),
               "invalid type_id")
 
+def has_special_memory_pressure(TYPE):
+    if TYPE._is_varsize():
+        return False
+    T = TYPE
+    while True:
+        if 'special_memory_pressure' in T._flds:
+            return True
+        if 'super' not in T._flds:
+            return False
+        T = T._flds['super']
+
+def get_memory_pressure_ofs(TYPE):
+    T = TYPE
+    while True:
+        if 'special_memory_pressure' in T._flds:
+            return llmemory.offsetof(T, 'special_memory_pressure')
+        if 'super' not in T._flds:
+            assert False, "get_ and has_memory_pressure disagree"
+        T = T._flds['super']    
 
 def encode_type_shape(builder, info, TYPE, index):
     """Encode the shape of the TYPE into the TYPE_INFO structure 'info'."""
@@ -202,12 +241,18 @@
         infobits |= T_HAS_GCPTR
     #
     fptrs = builder.special_funcptr_for_type(TYPE)
-    if fptrs:
+    if fptrs or has_special_memory_pressure(TYPE):
+        customdata = lltype.malloc(GCData.CUSTOM_DATA_STRUCT, flavor='raw',
+                                   immortal=True)
+        info.customdata = customdata
         if "destructor" in fptrs:
-            info.customfunc = fptrs["destructor"]
+            customdata.customfunc = fptrs["destructor"]
         if "old_style_finalizer" in fptrs:
-            info.customfunc = fptrs["old_style_finalizer"]
+            customdata.customfunc = fptrs["old_style_finalizer"]
             infobits |= T_HAS_OLDSTYLE_FINALIZER
+        if has_special_memory_pressure(TYPE):
+            infobits |= T_HAS_MEMORY_PRESSURE
+            info.customdata.memory_pressure_offset = get_memory_pressure_ofs(TYPE)
     #
     if not TYPE._is_varsize():
         info.fixedsize = llarena.round_up_for_allocation(
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -650,10 +650,12 @@
     else:
         return id(gcref._x)
 
-TOTAL_MEMORY, = range(1)
+TOTAL_MEMORY, TOTAL_MEMORY_PRESSURE = range(2)
 
 @not_rpython
 def get_stats(stat_no):
+    """ Long docstring goes here
+    """
     raise NotImplementedError
 
 @not_rpython
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -475,6 +475,13 @@
         self.lowleveltype = Ptr(self.object_type)
         self.gcflavor = gcflavor
 
+    def has_special_memory_pressure(self, tp):
+        if 'special_memory_pressure' in tp._flds:
+            return True
+        if 'super' in tp._flds:
+            return self.has_special_memory_pressure(tp._flds['super'])
+        return False
+
     def _setup_repr(self, llfields=None, hints=None, adtmeths=None):
         # NOTE: don't store mutable objects like the dicts below on 'self'
         #       before they are fully built, to avoid strange bugs in case
@@ -525,7 +532,10 @@
 
             bookkeeper = self.rtyper.annotator.bookkeeper
             if self.classdef in bookkeeper.memory_pressure_types:
-                llfields = [('special_memory_pressure', lltype.Signed)] + llfields
+                # we don't need to add it if it's already there for some of
+                # the parent type
+                if not self.has_special_memory_pressure(self.rbase.object_type):
+                    llfields.append(('special_memory_pressure', lltype.Signed))
 
             object_type = MkStruct(self.classdef.name,
                                    ('super', self.rbase.object_type),
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1624,12 +1624,16 @@
                 am3 = am2
                 am2 = am1
                 am1 = A()
+            am1 = am2 = am3 = None
             # what can we use for the res?
-            return rgc.get_stats(rgc.TOTAL_MEMORY)
+            for i in range(10):
+                gc.collect()
+            return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
         return f
 
     def test_nongc_opaque_attached_to_gc(self):
         res = self.run("nongc_opaque_attached_to_gc")
+        # the res is 0 for non-memory-pressure-accounting GC
         assert res == 0
 
     def define_limited_memory(self):

From pypy.commits at gmail.com  Fri Sep 29 07:10:20 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 04:10:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix Cython issue with classmethods in
 cdef classes
Message-ID: <59ce2a1c.c59c1c0a.ba86d.a69e@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92502:484db8c81ec1
Date: 2017-09-29 12:17 +0200
http://bitbucket.org/pypy/pypy/changeset/484db8c81ec1/

Log:	fix Cython issue with classmethods in cdef classes

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -654,7 +654,7 @@
         'PyClass_Type': 'space.gettypeobject(W_ClassObject.typedef)',
         'PyStaticMethod_Type': 'space.gettypeobject(StaticMethod.typedef)',
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
-        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)'
+        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
 
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -10,8 +10,7 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL,
-    PyTypeObjectPtr, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -109,7 +108,7 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        w_objclass = self.w_objclass 
+        w_objclass = self.w_objclass
         assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
             self.name, w_objclass.name))
@@ -117,7 +116,7 @@
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' of '%s' object needs an argument",
@@ -125,7 +124,7 @@
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%s' requires a '%s' object but received a '%T'",
@@ -333,11 +332,15 @@
 def PyClassMethod_New(space, w_func):
     return ClassMethod(w_func)
 
- at cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewMethod(space, w_type, method):
     return W_PyCMethodObject(space, method, w_type)
 
- at cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewClassMethod(space, w_type, method):
     return W_PyCClassMethodObject(space, method, w_type)
 
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,6 +83,22 @@
     return cls;
 }
 
+PyObject* make_classmethod(PyObject* method)
+{
+    // adapted from __Pyx_Method_ClassMethod
+    if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) {
+        return PyClassMethod_New(method);
+    }
+    else if (PyMethod_Check(method)) {
+        return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
+    }
+    else {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
+    }
+}
+
 static PyObject *
 foo_unset(fooobject *self)
 {
@@ -95,6 +111,7 @@
     {"copy",      (PyCFunction)foo_copy,      METH_NOARGS,  NULL},
     {"create",    (PyCFunction)foo_create,    METH_NOARGS|METH_STATIC,  NULL},
     {"classmeth", (PyCFunction)foo_classmeth, METH_NOARGS|METH_CLASS,  NULL},
+    {"fake_classmeth", (PyCFunction)foo_classmeth, METH_NOARGS,  NULL},
     {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL},
     {NULL, NULL}                 /* sentinel */
 };
@@ -167,19 +184,19 @@
     /* copied from numpy scalartypes.c for inherited classes */
     if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
     {
-        PyTypeObject *sup; 
-        /* We are inheriting from a Python type as well so 
+        PyTypeObject *sup;
+        /* We are inheriting from a Python type as well so
            give it first dibs on conversion */
         sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
-        /* Prevent recursion */ 
-        if (new_fooType != sup->tp_new) 
+        /* Prevent recursion */
+        if (new_fooType != sup->tp_new)
         {
             o = sup->tp_new(t, args, kwds);
             return o;
         }
     }
     o = t->tp_alloc(t, 0);
-    return o;   
+    return o;
 };
 
 static PyMemberDef foo_members[] = {
@@ -714,7 +731,7 @@
     "foo",
     "Module Doc",
     -1,
-    foo_functions, 
+    foo_functions,
     NULL,
     NULL,
     NULL,
@@ -748,6 +765,7 @@
 #endif
 {
     PyObject *d;
+    PyObject *fake_classmeth, *classmeth;
 #if PY_MAJOR_VERSION >= 3
     PyObject *module = PyModule_Create(&moduledef);
 #else
@@ -805,6 +823,10 @@
         INITERROR;
     gettype2 = PyObject_New(PyObject, &GetType2);
 
+    fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
+    classmeth = make_classmethod(fake_classmeth);
+    if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
+        INITERROR;
 
     d = PyModule_GetDict(module);
     if (d == NULL)
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -130,6 +130,12 @@
         assert repr(descr) == "<method 'copy' of 'foo.foo' objects>"
         raises(TypeError, descr, None)
 
+    def test_cython_fake_classmethod(self):
+        module = self.import_module(name='foo')
+        print(module.fooType.fake_classmeth)
+        print(type(module.fooType.fake_classmeth))
+        assert module.fooType.fake_classmeth() is module.fooType
+
     def test_new(self):
         # XXX cpython segfaults but if run singly (with -k test_new) this passes
         module = self.import_module(name='foo')

From pypy.commits at gmail.com  Fri Sep 29 09:23:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 06:23:38 -0700 (PDT)
Subject: [pypy-commit] pypy default: (antocuni, arigo)  Issue #2666
Message-ID: <59ce495a.048e1c0a.f5c1c.7057@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92503:3f4fc7771154
Date: 2017-09-29 15:23 +0200
http://bitbucket.org/pypy/pypy/changeset/3f4fc7771154/

Log:	(antocuni, arigo) Issue #2666

	Mess with cpyext. See comments.

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1362,6 +1362,47 @@
         assert Asize == Bsize
         assert Asize > basesize
 
+    def test_multiple_inheritance_bug1(self):
+        module = self.import_extension('foo', [
+           ("get_type", "METH_NOARGS",
+            '''
+                Py_INCREF(&Foo_Type);
+                return (PyObject *)&Foo_Type;
+            '''
+            ), ("forty_two", "METH_O",
+            '''
+                return PyInt_FromLong(42);
+            '''
+            )], prologue='''
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            static PyObject *dummy_new(PyTypeObject *t, PyObject *a,
+                                       PyObject *k)
+            {
+                abort();   /* never actually called in CPython */
+            }
+            ''', more_init = '''
+                Foo_Type.tp_base = (PyTypeObject *)PyExc_Exception;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = dummy_new;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
+            ''')
+        Foo = module.get_type()
+        class A(Foo, SyntaxError):
+            pass
+        assert A.__base__ is SyntaxError
+        A(42)    # assert is not aborting
+
+        class Bar(Exception):
+            __new__ = module.forty_two
+
+        class B(Bar, SyntaxError):
+            pass
+
+        assert B() == 42
+
 
 class AppTestHashable(AppTestCpythonExtensionBase):
     def test_unhashable(self):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -394,6 +394,9 @@
     ptr = get_new_method_def(space)
     ptr.c_ml_meth = rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 
+def is_tp_new_wrapper(space, ml):
+    return ml.c_ml_meth == rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
+
 def add_tp_new_wrapper(space, dict_w, pto):
     if "__new__" in dict_w:
         return
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -626,6 +626,12 @@
             if w_newdescr is None:    # see test_crash_mro_without_object_1
                 raise oefmt(space.w_TypeError, "cannot create '%N' instances",
                             self)
+            #
+            # issue #2666
+            if space.config.objspace.usemodules.cpyext:
+                w_newtype, w_newdescr = self.hack_which_new_to_call(
+                    w_newtype, w_newdescr)
+            #
             w_newfunc = space.get(w_newdescr, self)
             if (space.config.objspace.std.newshortcut and
                 not we_are_jitted() and
@@ -646,6 +652,30 @@
                                 "__init__() should return None")
         return w_newobject
 
+    def hack_which_new_to_call(self, w_newtype, w_newdescr):
+        # issue #2666: for cpyext, we need to hack in order to reproduce
+        # an "optimization" of CPython that actually changes behaviour
+        # in corner cases.
+        #
+        # * Normally, we use the __new__ found in the MRO in the normal way.
+        #
+        # * If by chance this __new__ happens to be implemented as a C
+        #   function, then instead, we discard it and use directly
+        #   self.__base__.tp_new.
+        #
+        # * Most of the time this is the same (and faster for CPython), but
+        #   it can fail if self.__base__ happens not to be the first base.
+        #
+        from pypy.module.cpyext.methodobject import W_PyCFunctionObject
+        from pypy.module.cpyext.typeobject import is_tp_new_wrapper
+
+        if (isinstance(w_newdescr, W_PyCFunctionObject) and
+                is_tp_new_wrapper(self.space, w_newdescr.ml)):
+            w_bestbase = find_best_base(self.bases_w)
+            return w_bestbase.lookup_where('__new__')
+        else:
+            return w_newtype, w_newdescr
+
     def descr_repr(self, space):
         w_mod = self.get_module()
         if w_mod is None or not space.isinstance_w(w_mod, space.w_text):

From pypy.commits at gmail.com  Fri Sep 29 10:36:26 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 07:36:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add a PyMethodDescr_Check() (not in the
 CPython API) for Cython's benefit
Message-ID: <59ce5a6a.d8b8df0a.f4373.70db@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92504:5acb984186b5
Date: 2017-09-29 16:33 +0200
http://bitbucket.org/pypy/pypy/changeset/5acb984186b5/

Log:	Add a PyMethodDescr_Check() (not in the CPython API) for Cython's
	benefit

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,8 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts,
+    build_type_checkers)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -136,6 +137,10 @@
         ret = self.call(space, w_instance, w_args, w_kw)
         return ret
 
+# PyPy addition, for Cython
+_, _ = build_type_checkers("MethodDescr", W_PyCMethodObject)
+
+
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
     from pypy.interpreter.function import BuiltinFunction
@@ -162,6 +167,7 @@
                             (self.name, self.w_objclass.getname(self.space)))
 
 
+
 class W_PyCWrapperObject(W_Root):
     def __init__(self, space, pto, method_name, wrapper_func,
                  wrapper_func_kwds, doc, func, offset=None):
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,19 +83,32 @@
     return cls;
 }
 
+// for CPython
+#ifndef PyMethodDescr_Check
+int PyMethodDescr_Check(PyObject* method)
+{
+    PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
+    if (!meth) return 0;
+    int res = PyObject_TypeCheck(method, meth->ob_type);
+    Py_DECREF(meth);
+    return res;
+}
+#endif
+
 PyObject* make_classmethod(PyObject* method)
 {
     // adapted from __Pyx_Method_ClassMethod
-    if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) {
-        return PyClassMethod_New(method);
+    if (PyMethodDescr_Check(method)) {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
     }
     else if (PyMethod_Check(method)) {
         return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
     }
     else {
-        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
-        PyTypeObject *d_type = descr->d_type;
-        return PyDescr_NewClassMethod(d_type, descr->d_method);
+        PyErr_SetString(PyExc_TypeError, "unknown method kind");
+        return NULL;
     }
 }
 
@@ -825,6 +838,8 @@
 
     fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
     classmeth = make_classmethod(fake_classmeth);
+    if (classmeth == NULL)
+        INITERROR;
     if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
         INITERROR;
 

From pypy.commits at gmail.com  Fri Sep 29 11:15:01 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 08:15:01 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59ce6375.e186df0a.3dbb3.2cb0@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92505:a0fe3f649ba8
Date: 2017-09-29 17:14 +0200
http://bitbucket.org/pypy/pypy/changeset/a0fe3f649ba8/

Log:	hg merge default

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,8 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts,
+    build_type_checkers)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -129,6 +130,10 @@
         ret = self.call(space, w_instance, w_args, w_kw)
         return ret
 
+# PyPy addition, for Cython
+_, _ = build_type_checkers("MethodDescr", W_PyCMethodObject)
+
+
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
     from pypy.interpreter.function import BuiltinFunction
@@ -155,6 +160,7 @@
             (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
 
 
+
 class W_PyCWrapperObject(W_Root):
     def __init__(self, space, pto, method_name, wrapper_func,
                  wrapper_func_kwds, doc, func, offset=None):
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,19 +83,32 @@
     return cls;
 }
 
+// for CPython
+#ifndef PyMethodDescr_Check
+int PyMethodDescr_Check(PyObject* method)
+{
+    PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
+    if (!meth) return 0;
+    int res = PyObject_TypeCheck(method, meth->ob_type);
+    Py_DECREF(meth);
+    return res;
+}
+#endif
+
 PyObject* make_classmethod(PyObject* method)
 {
     // adapted from __Pyx_Method_ClassMethod
-    if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) {
-        return PyClassMethod_New(method);
+    if (PyMethodDescr_Check(method)) {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_common.d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
     }
     else if (PyMethod_Check(method)) {
         return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
     }
     else {
-        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
-        PyTypeObject *d_type = descr->d_common.d_type;
-        return PyDescr_NewClassMethod(d_type, descr->d_method);
+        PyErr_SetString(PyExc_TypeError, "unknown method kind");
+        return NULL;
     }
 }
 
@@ -828,6 +841,8 @@
 
     fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
     classmeth = make_classmethod(fake_classmeth);
+    if (classmeth == NULL)
+        INITERROR;
     if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
         INITERROR;
 
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1321,6 +1321,47 @@
         assert Asize == Bsize
         assert Asize > basesize
 
+    def test_multiple_inheritance_bug1(self):
+        module = self.import_extension('foo', [
+           ("get_type", "METH_NOARGS",
+            '''
+                Py_INCREF(&Foo_Type);
+                return (PyObject *)&Foo_Type;
+            '''
+            ), ("forty_two", "METH_O",
+            '''
+                return PyLong_FromLong(42);
+            '''
+            )], prologue='''
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            static PyObject *dummy_new(PyTypeObject *t, PyObject *a,
+                                       PyObject *k)
+            {
+                abort();   /* never actually called in CPython */
+            }
+            ''', more_init = '''
+                Foo_Type.tp_base = (PyTypeObject *)PyExc_Exception;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = dummy_new;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
+            ''')
+        Foo = module.get_type()
+        class A(Foo, SyntaxError):
+            pass
+        assert A.__base__ is SyntaxError
+        A(42)    # assert is not aborting
+
+        class Bar(Exception):
+            __new__ = module.forty_two
+
+        class B(Bar, SyntaxError):
+            pass
+
+        assert B() == 42
+
 
 class AppTestHashable(AppTestCpythonExtensionBase):
     def test_unhashable(self):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -395,6 +395,9 @@
     ptr = get_new_method_def(space)
     ptr.c_ml_meth = rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 
+def is_tp_new_wrapper(space, ml):
+    return ml.c_ml_meth == rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
+
 def add_tp_new_wrapper(space, dict_w, pto):
     if "__new__" in dict_w:
         return
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -641,6 +641,12 @@
             if w_newdescr is None:    # see test_crash_mro_without_object_1
                 raise oefmt(space.w_TypeError, "cannot create '%N' instances",
                             self)
+            #
+            # issue #2666
+            if space.config.objspace.usemodules.cpyext:
+                w_newtype, w_newdescr = self.hack_which_new_to_call(
+                    w_newtype, w_newdescr)
+            #
             w_newfunc = space.get(w_newdescr, space.w_None, w_type=self)
             if (space.config.objspace.std.newshortcut and
                 not we_are_jitted() and
@@ -661,6 +667,30 @@
                                 "__init__() should return None")
         return w_newobject
 
+    def hack_which_new_to_call(self, w_newtype, w_newdescr):
+        # issue #2666: for cpyext, we need to hack in order to reproduce
+        # an "optimization" of CPython that actually changes behaviour
+        # in corner cases.
+        #
+        # * Normally, we use the __new__ found in the MRO in the normal way.
+        #
+        # * If by chance this __new__ happens to be implemented as a C
+        #   function, then instead, we discard it and use directly
+        #   self.__base__.tp_new.
+        #
+        # * Most of the time this is the same (and faster for CPython), but
+        #   it can fail if self.__base__ happens not to be the first base.
+        #
+        from pypy.module.cpyext.methodobject import W_PyCFunctionObject
+        from pypy.module.cpyext.typeobject import is_tp_new_wrapper
+
+        if (isinstance(w_newdescr, W_PyCFunctionObject) and
+                is_tp_new_wrapper(self.space, w_newdescr.ml)):
+            w_bestbase = find_best_base(self.bases_w)
+            return w_bestbase.lookup_where('__new__')
+        else:
+            return w_newtype, w_newdescr
+
     def descr_repr(self, space):
         w_mod = self.get_module()
         if w_mod is None or not space.isinstance_w(w_mod, space.w_text):

From pypy.commits at gmail.com  Fri Sep 29 11:20:01 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 29 Sep 2017 08:20:01 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: * Review all the places that
 add memory pressure outside of numpy and cpyext
Message-ID: <59ce64a1.2c97df0a.e6a32.3581@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92506:f8fd62a0f087
Date: 2017-09-29 17:19 +0200
http://bitbucket.org/pypy/pypy/changeset/f8fd62a0f087/

Log:	* Review all the places that add memory pressure outside of numpy
	and cpyext
	* Add a test
	* Fixes about cast_pointer in the presence of subclasses
	* Write down the app-level interface to that

diff --git a/lib_pypy/_sqlite3.py b/lib_pypy/_sqlite3.py
--- a/lib_pypy/_sqlite3.py
+++ b/lib_pypy/_sqlite3.py
@@ -35,7 +35,7 @@
 except ImportError:
     assert '__pypy__' not in sys.builtin_module_names
     newlist_hint = lambda sizehint: []
-    add_memory_pressure = lambda size: None
+    add_memory_pressure = lambda size, obj: None
 
 if sys.version_info[0] >= 3:
     StandardError = Exception
@@ -153,9 +153,10 @@
     factory = Connection if not factory else factory
     # an sqlite3 db seems to be around 100 KiB at least (doesn't matter if
     # backed by :memory: or a file)
-    add_memory_pressure(100 * 1024)
-    return factory(database, timeout, detect_types, isolation_level,
+    res = factory(database, timeout, detect_types, isolation_level,
                     check_same_thread, factory, cached_statements)
+    add_memory_pressure(100 * 1024, res)
+    return res
 
 
 def _unicode_text_factory(x):
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -142,11 +142,14 @@
                   space.newbool(debug))
 
 @unwrap_spec(estimate=int)
-def add_memory_pressure(estimate):
+def add_memory_pressure(space, estimate, w_obj=None):
     """ Add memory pressure of estimate bytes. Useful when calling a C function
     that internally allocates a big chunk of memory. This instructs the GC to
     garbage collect sooner than it would otherwise."""
-    rgc.add_memory_pressure(estimate)
+    if space.is_none(w_obj):
+        rgc.add_memory_pressure(estimate)
+    else:
+        rgc.add_memory_pressure(estimate, w_obj)
 
 @unwrap_spec(w_frame=PyFrame)
 def locals_to_fast(space, w_frame):
diff --git a/pypy/module/_cffi_backend/allocator.py b/pypy/module/_cffi_backend/allocator.py
--- a/pypy/module/_cffi_backend/allocator.py
+++ b/pypy/module/_cffi_backend/allocator.py
@@ -21,13 +21,13 @@
         if self.w_alloc is None:
             if self.should_clear_after_alloc:
                 ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
-                                    flavor='raw', zero=True,
-                                    add_memory_pressure=True)
+                                    flavor='raw', zero=True)
             else:
                 ptr = lltype.malloc(rffi.CCHARP.TO, datasize,
-                                    flavor='raw', zero=False,
-                                    add_memory_pressure=True)
-            return cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+                                    flavor='raw', zero=False)
+            w_res = cdataobj.W_CDataNewStd(space, ptr, ctype, length)
+            rgc.add_memory_pressure(datasize, w_res)
+            return w_res
         else:
             w_raw_cdata = space.call_function(self.w_alloc,
                                               space.newint(datasize))
@@ -53,7 +53,7 @@
             if self.w_free is not None:
                 res.w_free = self.w_free
                 res.register_finalizer(space)
-            rgc.add_memory_pressure(datasize)
+            rgc.add_memory_pressure(datasize, res)
             return res
 
     @unwrap_spec(w_init=WrappedDefault(None))
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -447,7 +447,7 @@
             with self as ptr:
                 w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
         if size != 0:
-            rgc.add_memory_pressure(size)
+            rgc.add_memory_pressure(size, w_res)
         return w_res
 
     def unpack(self, length):
diff --git a/pypy/module/_hashlib/interp_hashlib.py b/pypy/module/_hashlib/interp_hashlib.py
--- a/pypy/module/_hashlib/interp_hashlib.py
+++ b/pypy/module/_hashlib/interp_hashlib.py
@@ -61,7 +61,8 @@
         ctx = ropenssl.EVP_MD_CTX_new()
         if ctx is None:
             raise MemoryError
-        rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size)
+        rgc.add_memory_pressure(ropenssl.HASH_MALLOC_SIZE + self.digest_size,
+                                self)
         try:
             if copy_from:
                 if not ropenssl.EVP_MD_CTX_copy(ctx, copy_from):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1315,8 +1315,8 @@
         if not ctx:
             raise ssl_error(space, "failed to allocate SSL context")
 
-        rgc.add_memory_pressure(10 * 1024)
         self = space.allocate_instance(_SSLContext, w_subtype)
+        rgc.add_memory_pressure(10 * 1024, self)
         self.ctx = ctx
         self.check_hostname = False
         self.register_finalizer(space)
diff --git a/pypy/module/gc/__init__.py b/pypy/module/gc/__init__.py
--- a/pypy/module/gc/__init__.py
+++ b/pypy/module/gc/__init__.py
@@ -28,6 +28,7 @@
                 'get_objects': 'referents.get_objects',
                 'get_referents': 'referents.get_referents',
                 'get_referrers': 'referents.get_referrers',
+                'get_stats': 'referents.get_stats',
                 '_dump_rpy_heap': 'referents._dump_rpy_heap',
                 'get_typeids_z': 'referents.get_typeids_z',
                 'get_typeids_list': 'referents.get_typeids_list',
diff --git a/pypy/module/gc/referents.py b/pypy/module/gc/referents.py
--- a/pypy/module/gc/referents.py
+++ b/pypy/module/gc/referents.py
@@ -1,6 +1,6 @@
 from rpython.rlib import rgc
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef
+from pypy.interpreter.typedef import TypeDef, interp_attrproperty
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.error import oefmt, wrap_oserror
 from rpython.rlib.objectmodel import we_are_translated
@@ -170,3 +170,15 @@
     l = rgc.get_typeids_list()
     list_w = [space.newint(l[i]) for i in range(len(l))]
     return space.newlist(list_w)
+
+class W_GcStats(W_Root):
+    def __init__(self):
+        self.total_memory_pressure = rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+
+W_GcStats.typedef = TypeDef("GcStats",
+    total_memory_pressure=interp_attrproperty("total_memory_pressure",
+        cls=W_GcStats, wrapfn="newint"),
+)
+
+def get_stats(space):
+    return W_GcStats()
diff --git a/pypy/module/pyexpat/interp_pyexpat.py b/pypy/module/pyexpat/interp_pyexpat.py
--- a/pypy/module/pyexpat/interp_pyexpat.py
+++ b/pypy/module/pyexpat/interp_pyexpat.py
@@ -840,11 +840,11 @@
     # Currently this is just the size of the pointer and some estimated bytes.
     # The struct isn't actually defined in expat.h - it is in xmlparse.c
     # XXX: find a good estimate of the XML_ParserStruct
-    rgc.add_memory_pressure(XML_Parser_SIZE + 300)
     if not xmlparser:
         raise oefmt(space.w_RuntimeError, "XML_ParserCreate failed")
 
     parser = W_XMLParserType(space, xmlparser, w_intern)
+    rgc.add_memory_pressure(XML_Parser_SIZE + 300, parser)
     XML_SetUnknownEncodingHandler(
         parser.itself, UnknownEncodingHandlerData_callback,
         rffi.cast(rffi.VOIDP, parser.id))
diff --git a/rpython/jit/codewriter/support.py b/rpython/jit/codewriter/support.py
--- a/rpython/jit/codewriter/support.py
+++ b/rpython/jit/codewriter/support.py
@@ -675,6 +675,8 @@
 
     def _ll_1_gc_add_memory_pressure(num):
         llop.gc_add_memory_pressure(lltype.Void, num)
+    def _ll_2_gc_add_memory_pressure(num, obj):
+        llop.gc_add_memory_pressure(lltype.Void, num, obj)
 
 
 def setup_extra_builtin(rtyper, oopspec_name, nb_args, extra=None):
diff --git a/rpython/memory/gc/inspector.py b/rpython/memory/gc/inspector.py
--- a/rpython/memory/gc/inspector.py
+++ b/rpython/memory/gc/inspector.py
@@ -2,6 +2,7 @@
 Utility RPython functions to inspect objects in the GC.
 """
 from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, llgroup
+from rpython.rtyper.lltypesystem.lloperation import llop
 from rpython.rlib.objectmodel import free_non_gc_object
 from rpython.rlib import rposix, rgc, jit
 
@@ -187,6 +188,11 @@
             ofs = gc.get_memory_pressure_ofs(typeid)
             val = (obj + ofs).signed[0]
             self.count += val
+        gc.trace(obj, self._ref, None)
+
+    def _ref(self, pointer, _):
+        obj = pointer.address[0]
+        self.add(obj)
 
 
 class HeapDumper(BaseWalker):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -839,12 +839,21 @@
     gct_fv_gc_malloc_varsize = gct_fv_gc_malloc
 
     def gct_gc_add_memory_pressure(self, hop):
+        def _find_correct_type(TP):
+            T = TP.TO
+            while 'special_memory_pressure' not in T._flds:
+                T = T._flds['super']
+            return T
+
         if hasattr(self, 'raw_malloc_memory_pressure_ptr'):
             op = hop.spaceop
             size = op.args[0]
             if len(op.args) == 2:
                 v_fld = rmodel.inputconst(lltype.Void, "special_memory_pressure")
-                hop.genop("bare_setfield", [op.args[1], v_fld, size])
+                T = _find_correct_type(op.args[1].concretetype)
+                v_inst = hop.genop("cast_pointer", [op.args[1]],
+                    resulttype=lltype.Ptr(T))
+                hop.genop("bare_setfield", [v_inst, v_fld, size])
                 v_adr = hop.genop("cast_ptr_to_adr", [op.args[1]],
                     resulttype=llmemory.Address)
             else:
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -85,7 +85,11 @@
 
 
 def allocate_lock():
-    return Lock(allocate_ll_lock())
+    # Add some memory pressure for the size of the lock because it is an
+    # Opaque object
+    lock = Lock(allocate_ll_lock())
+    rgc.add_memory_pressure(TLOCKP_SIZE, lock)
+    return lock
 
 @specialize.arg(0)
 def ll_start_new_thread(func):
@@ -242,9 +246,6 @@
     if rffi.cast(lltype.Signed, res) <= 0:
         lltype.free(ll_lock, flavor='raw', track_allocation=False)
         raise error("out of resources")
-    # Add some memory pressure for the size of the lock because it is an
-    # Opaque object
-    rgc.add_memory_pressure(TLOCKP_SIZE)
     return ll_lock
 
 def free_ll_lock(ll_lock):
diff --git a/rpython/rlib/rzlib.py b/rpython/rlib/rzlib.py
--- a/rpython/rlib/rzlib.py
+++ b/rpython/rlib/rzlib.py
@@ -269,7 +269,6 @@
     compress data.
     """
     stream = lltype.malloc(z_stream, flavor='raw', zero=True)
-    rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _deflateInit2(stream, level, method, wbits, memLevel, strategy)
     if err == Z_OK:
         if zdict is not None:
@@ -304,7 +303,6 @@
     decompress data.
     """
     stream = lltype.malloc(z_stream, flavor='raw', zero=True)
-    rgc.add_memory_pressure(rffi.sizeof(z_stream))
     err = _inflateInit2(stream, wbits)
     if err == Z_OK:
         if zdict is not None and wbits < 0:
diff --git a/rpython/translator/c/test/test_newgc.py b/rpython/translator/c/test/test_newgc.py
--- a/rpython/translator/c/test/test_newgc.py
+++ b/rpython/translator/c/test/test_newgc.py
@@ -1672,6 +1672,38 @@
 class TestIncrementalMiniMarkGC(TestMiniMarkGC):
     gcpolicy = "incminimark"
 
+    def define_total_memory_pressure(cls):
+        class A(object):
+            def __init__(self):
+                rgc.add_memory_pressure(10, self)
+
+            def __del__(self):
+                pass
+
+        class B(A):
+            def __init__(self):
+                rgc.add_memory_pressure(10, self)
+
+        class C(A):
+            pass
+
+        class Glob(object):
+            pass
+        glob = Glob()
+
+        def f():
+            glob.l = [None] * 3
+            for i in range(10000):
+                glob.l[i % 3] = A()
+                glob.l[(i + 1) % 3] = B()
+                glob.l[(i + 2) % 3] = C()
+            return rgc.get_stats(rgc.TOTAL_MEMORY_PRESSURE)
+        return f
+
+    def test_total_memory_pressure(self):
+        res = self.run("total_memory_pressure")
+        assert res == 30 # total reachable is 3
+
     def define_random_pin(self):
         class A:
             foo = None

From pypy.commits at gmail.com  Fri Sep 29 11:53:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 08:53:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: (antocuni, arigo)
Message-ID: <59ce6c6d.090b1c0a.5e7af.2af2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92507:075a4ef2fb28
Date: 2017-09-29 17:48 +0200
http://bitbucket.org/pypy/pypy/changeset/075a4ef2fb28/

Log:	(antocuni, arigo)

	oops, fix for 3f4fc7771154 (some tests in cpyext fail)

diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -670,6 +670,7 @@
         from pypy.module.cpyext.typeobject import is_tp_new_wrapper
 
         if (isinstance(w_newdescr, W_PyCFunctionObject) and
+                w_newtype is not self and
                 is_tp_new_wrapper(self.space, w_newdescr.ml)):
             w_bestbase = find_best_base(self.bases_w)
             return w_bestbase.lookup_where('__new__')

From pypy.commits at gmail.com  Fri Sep 29 11:53:19 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 08:53:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge heads
Message-ID: <59ce6c6f.579c1c0a.19e0.7dc1@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92508:c4d6cc4a81fe
Date: 2017-09-29 17:52 +0200
http://bitbucket.org/pypy/pypy/changeset/c4d6cc4a81fe/

Log:	merge heads

diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -10,7 +10,8 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts,
+    build_type_checkers)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -136,6 +137,10 @@
         ret = self.call(space, w_instance, w_args, w_kw)
         return ret
 
+# PyPy addition, for Cython
+_, _ = build_type_checkers("MethodDescr", W_PyCMethodObject)
+
+
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
     from pypy.interpreter.function import BuiltinFunction
@@ -162,6 +167,7 @@
                             (self.name, self.w_objclass.getname(self.space)))
 
 
+
 class W_PyCWrapperObject(W_Root):
     def __init__(self, space, pto, method_name, wrapper_func,
                  wrapper_func_kwds, doc, func, offset=None):
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,19 +83,32 @@
     return cls;
 }
 
+// for CPython
+#ifndef PyMethodDescr_Check
+int PyMethodDescr_Check(PyObject* method)
+{
+    PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
+    if (!meth) return 0;
+    int res = PyObject_TypeCheck(method, meth->ob_type);
+    Py_DECREF(meth);
+    return res;
+}
+#endif
+
 PyObject* make_classmethod(PyObject* method)
 {
     // adapted from __Pyx_Method_ClassMethod
-    if (PyObject_TypeCheck(method, &PyWrapperDescr_Type)) {
-        return PyClassMethod_New(method);
+    if (PyMethodDescr_Check(method)) {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
     }
     else if (PyMethod_Check(method)) {
         return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
     }
     else {
-        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
-        PyTypeObject *d_type = descr->d_type;
-        return PyDescr_NewClassMethod(d_type, descr->d_method);
+        PyErr_SetString(PyExc_TypeError, "unknown method kind");
+        return NULL;
     }
 }
 
@@ -825,6 +838,8 @@
 
     fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
     classmeth = make_classmethod(fake_classmeth);
+    if (classmeth == NULL)
+        INITERROR;
     if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
         INITERROR;
 

From pypy.commits at gmail.com  Fri Sep 29 12:33:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 09:33:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: (antocuni, arigo)
Message-ID: <59ce75db.4cb0df0a.8f3d.b536@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r92509:094524b8baa4
Date: 2017-09-29 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/094524b8baa4/

Log:	(antocuni, arigo)

	aaaaaa even more hacking

diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1403,6 +1403,11 @@
 
         assert B() == 42
 
+        # aaaaa even more hackiness
+        class C(A):
+            pass
+        C(42)   # assert is not aborting
+
 
 class AppTestHashable(AppTestCpythonExtensionBase):
     def test_unhashable(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -667,15 +667,30 @@
         #   it can fail if self.__base__ happens not to be the first base.
         #
         from pypy.module.cpyext.methodobject import W_PyCFunctionObject
+
+        if isinstance(w_newdescr, W_PyCFunctionObject):
+            return self._really_hack_which_new_to_call(w_newtype, w_newdescr)
+        else:
+            return w_newtype, w_newdescr
+
+    def _really_hack_which_new_to_call(self, w_newtype, w_newdescr):
+        # This logic is moved in yet another helper function that
+        # is recursive.  We call this only if we see a
+        # W_PyCFunctionObject.  That's a performance optimization
+        # because in the common case, we won't call any function that
+        # contains the stack checks.
+        from pypy.module.cpyext.methodobject import W_PyCFunctionObject
         from pypy.module.cpyext.typeobject import is_tp_new_wrapper
 
         if (isinstance(w_newdescr, W_PyCFunctionObject) and
                 w_newtype is not self and
                 is_tp_new_wrapper(self.space, w_newdescr.ml)):
             w_bestbase = find_best_base(self.bases_w)
-            return w_bestbase.lookup_where('__new__')
-        else:
-            return w_newtype, w_newdescr
+            if w_bestbase is not None:
+                w_newtype, w_newdescr = w_bestbase.lookup_where('__new__')
+                return w_bestbase._really_hack_which_new_to_call(w_newtype,
+                                                                 w_newdescr)
+        return w_newtype, w_newdescr
 
     def descr_repr(self, space):
         w_mod = self.get_module()

From pypy.commits at gmail.com  Fri Sep 29 13:38:13 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 29 Sep 2017 10:38:13 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: whack for a bit until we a)
 initialize to 0 and b) have it not on W_Root
Message-ID: <59ce8505.57e11c0a.3240a.6194@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92510:4420cc75f88e
Date: 2017-09-29 19:37 +0200
http://bitbucket.org/pypy/pypy/changeset/4420cc75f88e/

Log:	whack for a bit until we a) initialize to 0 and b) have it not on
	W_Root

diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -146,10 +146,10 @@
     """ Add memory pressure of estimate bytes. Useful when calling a C function
     that internally allocates a big chunk of memory. This instructs the GC to
     garbage collect sooner than it would otherwise."""
-    if space.is_none(w_obj):
-        rgc.add_memory_pressure(estimate)
-    else:
-        rgc.add_memory_pressure(estimate, w_obj)
+    #if space.is_none(w_obj):
+    rgc.add_memory_pressure(estimate)
+    #else:
+    #    rgc.add_memory_pressure(estimate, w_obj)
 
 @unwrap_spec(w_frame=PyFrame)
 def locals_to_fast(space, w_frame):
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -447,7 +447,10 @@
             with self as ptr:
                 w_res = W_CDataGCP(space, ptr, self.ctype, self, w_destructor)
         if size != 0:
-            rgc.add_memory_pressure(size, w_res)
+            if isinstance(w_res, W_CDataGCP):
+                rgc.add_memory_pressure(size, w_res)
+            else:
+                rgc.add_memory_pressure(size, self)
         return w_res
 
     def unpack(self, length):
diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1316,6 +1316,7 @@
             raise ssl_error(space, "failed to allocate SSL context")
 
         self = space.allocate_instance(_SSLContext, w_subtype)
+        assert isinstance(self, _SSLContext)
         rgc.add_memory_pressure(10 * 1024, self)
         self.ctx = ctx
         self.check_hostname = False
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -15,6 +15,7 @@
     RuntimeTypeInfo, getRuntimeTypeInfo, typeOf, Void, FuncType, Bool, Signed,
     functionptr, attachRuntimeTypeInfo)
 from rpython.rtyper.lltypesystem.lloperation import llop
+from rpython.rtyper.llannotation import lltype_to_annotation
 from rpython.rtyper.llannotation import SomePtr
 from rpython.rtyper.lltypesystem import rstr
 from rpython.rtyper.rmodel import (
@@ -536,6 +537,9 @@
                 # the parent type
                 if not self.has_special_memory_pressure(self.rbase.object_type):
                     llfields.append(('special_memory_pressure', lltype.Signed))
+                    fields['special_memory_pressure'] = (
+                        'special_memory_pressure',
+                        self.rtyper.getrepr(lltype_to_annotation(lltype.Signed)))
 
             object_type = MkStruct(self.classdef.name,
                                    ('super', self.rbase.object_type),
@@ -677,6 +681,8 @@
         while base.classdef is not None:
             base = base.rbase
             for fieldname in base.fields:
+                if fieldname == 'special_memory_pressure':
+                    continue
                 try:
                     mangled, r = base._get_field(fieldname)
                 except KeyError:
@@ -731,6 +737,9 @@
                            resulttype=Ptr(self.object_type))
         ctypeptr = inputconst(CLASSTYPE, self.rclass.getvtable())
         self.setfield(vptr, '__class__', ctypeptr, llops)
+        if self.has_special_memory_pressure(self.object_type):
+            self.setfield(vptr, 'special_memory_pressure',
+                inputconst(lltype.Signed, 0), llops)
         # initialize instance attributes from their defaults from the class
         if self.classdef is not None:
             flds = self.allinstancefields.keys()

From pypy.commits at gmail.com  Fri Sep 29 13:46:44 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 10:46:44 -0700 (PDT)
Subject: [pypy-commit] pypy len_w: Branch to make "len_w()" a method on
 W_Root, for optimization purposes
Message-ID: <59ce8704.8284df0a.7fcb7.a3ca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: len_w
Changeset: r92511:6136622851dc
Date: 2017-09-29 19:36 +0200
http://bitbucket.org/pypy/pypy/changeset/6136622851dc/

Log:	Branch to make "len_w()" a method on W_Root, for optimization
	purposes


From pypy.commits at gmail.com  Fri Sep 29 13:46:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 10:46:46 -0700 (PDT)
Subject: [pypy-commit] pypy len_w: The idea is to do this change,
 to make space.len_w() actually
Message-ID: <59ce8706.fb87df0a.9a739.a5ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: len_w
Changeset: r92512:6bb19d21ddea
Date: 2017-09-29 19:46 +0200
http://bitbucket.org/pypy/pypy/changeset/6bb19d21ddea/

Log:	The idea is to do this change, to make space.len_w() actually not
	allocate a temporary W_IntObject even if the call to space.len_w()
	is not JITted.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -328,6 +328,11 @@
         raise oefmt(space.w_TypeError,
                     "ord() expected string of length 1, but %T found", self)
 
+    def len_w(self, space):
+        # NOTE: you still need to override __len__ in your specific
+        # subclass' typedef; this is only here for optimization.
+        return space.int_w(space.len(self))
+
     def spacebind(self, space):
         """ Return a version of the object bound to a specific object space
         instance. This is used for objects (like e.g. TypeDefs) that are
@@ -800,7 +805,7 @@
 
     def len_w(self, w_obj):
         """shortcut for space.int_w(space.len(w_obj))"""
-        return self.int_w(self.len(w_obj))
+        return w_obj.len_w(self)
 
     def contains_w(self, w_container, w_item):
         """shortcut for space.is_true(space.contains(w_container, w_item))"""
diff --git a/pypy/objspace/std/listobject.py b/pypy/objspace/std/listobject.py
--- a/pypy/objspace/std/listobject.py
+++ b/pypy/objspace/std/listobject.py
@@ -308,6 +308,9 @@
     def length(self):
         return self.strategy.length(self)
 
+    def len_w(self, space):
+        return self.length()
+
     def getitem(self, index):
         """Returns the wrapped object that is found in the
         list at the given index. The index must be unwrapped.
diff --git a/pypy/objspace/std/tupleobject.py b/pypy/objspace/std/tupleobject.py
--- a/pypy/objspace/std/tupleobject.py
+++ b/pypy/objspace/std/tupleobject.py
@@ -283,6 +283,9 @@
     def length(self):
         return len(self.wrappeditems)
 
+    def len_w(self, space):
+        return self.length()
+
     def descr_hash(self, space):
         if _unroll_condition(self):
             return self._descr_hash_unroll(space)

From pypy.commits at gmail.com  Fri Sep 29 13:48:36 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 29 Sep 2017 10:48:36 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-jit: hg merge default
Message-ID: <59ce8774.090b1c0a.5e7af.3fc3@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: cpyext-jit
Changeset: r92513:820724c1b021
Date: 2017-09-29 19:48 +0200
http://bitbucket.org/pypy/pypy/changeset/820724c1b021/

Log:	hg merge default

diff too long, truncating to 2000 out of 20222 lines

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -25,16 +25,17 @@
 ^pypy/module/cpyext/test/.+\.manifest$
 ^pypy/module/test_lib_pypy/ctypes_tests/.+\.o$
 ^pypy/module/test_lib_pypy/ctypes_tests/_ctypes_test\.o$
-^pypy/module/cppyy/src/.+\.o$
-^pypy/module/cppyy/bench/.+\.so$
-^pypy/module/cppyy/bench/.+\.root$
-^pypy/module/cppyy/bench/.+\.d$
-^pypy/module/cppyy/src/.+\.errors$
-^pypy/module/cppyy/test/.+_rflx\.cpp$
-^pypy/module/cppyy/test/.+\.so$
-^pypy/module/cppyy/test/.+\.rootmap$
-^pypy/module/cppyy/test/.+\.exe$
-^pypy/module/cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/src/.+\.o$
+^pypy/module/_cppyy/bench/.+\.so$
+^pypy/module/_cppyy/bench/.+\.root$
+^pypy/module/_cppyy/bench/.+\.d$
+^pypy/module/_cppyy/src/.+\.errors$
+^pypy/module/_cppyy/test/.+_rflx\.cpp$
+^pypy/module/_cppyy/test/.+\.so$
+^pypy/module/_cppyy/test/.+\.rootmap$
+^pypy/module/_cppyy/test/.+\.exe$
+^pypy/module/_cppyy/test/.+_cint.h$
+^pypy/module/_cppyy/.+/*\.pcm$
 ^pypy/module/test_lib_pypy/cffi_tests/__pycache__.+$
 ^pypy/doc/.+\.html$
 ^pypy/doc/config/.+\.rst$
@@ -88,6 +89,3 @@
 ^release/
 ^rpython/_cache$
 
-pypy/module/cppyy/.+/*\.pcm
-
-
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -60,8 +60,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -102,6 +102,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -110,14 +111,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -156,6 +156,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -182,11 +183,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -194,6 +197,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -204,6 +208,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -214,14 +219,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -243,7 +246,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -275,6 +277,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -305,6 +308,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -325,12 +329,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/Makefile b/Makefile
--- a/Makefile
+++ b/Makefile
@@ -10,7 +10,7 @@
 RUNINTERP = $(PYPY_EXECUTABLE)
 endif
 
-.PHONY: cffi_imports
+.PHONY: pypy-c cffi_imports
 
 pypy-c:
 	@echo
@@ -32,7 +32,7 @@
 	@echo "===================================================================="
 	@echo
 	@sleep 5
-	$(RUNINTERP) rpython/bin/rpython -Ojit pypy/goal/targetpypystandalone.py
+	cd pypy/goal && $(RUNINTERP) ../../rpython/bin/rpython -Ojit targetpypystandalone.py
 
 # Note: the -jN option, or MAKEFLAGS=-jN, are not usable.  They are
 # replaced with an opaque --jobserver option by the time this Makefile
@@ -40,4 +40,4 @@
 # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html
 
 cffi_imports: pypy-c
-	PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py || /bin/true
+	PYTHONPATH=. pypy/goal/pypy-c pypy/tool/build_cffi_imports.py || /bin/true
diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -361,17 +361,20 @@
 
         if handle is None:
             if flags & _FUNCFLAG_CDECL:
-                self._handle = _ffi.CDLL(name, mode)
+                pypy_dll = _ffi.CDLL(name, mode)
             else:
-                self._handle = _ffi.WinDLL(name, mode)
-        else:
-            self._handle = handle
+                pypy_dll = _ffi.WinDLL(name, mode)
+            self.__pypy_dll__ = pypy_dll
+            handle = int(pypy_dll)
+            if _sys.maxint > 2 ** 32:
+                handle = int(handle)   # long -> int
+        self._handle = handle
 
     def __repr__(self):
-        return "<%s '%s', handle %r at 0x%x>" % (
-            self.__class__.__name__, self._name, self._handle,
-            id(self) & (_sys.maxint * 2 + 1))
-
+        return "<%s '%s', handle %x at %x>" % \
+               (self.__class__.__name__, self._name,
+                (self._handle & (_sys.maxint*2 + 1)),
+                id(self) & (_sys.maxint*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
diff --git a/lib-python/2.7/ctypes/test/test_byteswap.py b/lib-python/2.7/ctypes/test/test_byteswap.py
--- a/lib-python/2.7/ctypes/test/test_byteswap.py
+++ b/lib-python/2.7/ctypes/test/test_byteswap.py
@@ -23,7 +23,6 @@
             setattr(bits, "i%s" % i, 1)
             dump(bits)
 
-    @xfail
     def test_endian_short(self):
         if sys.byteorder == "little":
             self.assertIs(c_short.__ctype_le__, c_short)
@@ -51,7 +50,6 @@
         self.assertEqual(bin(s), "3412")
         self.assertEqual(s.value, 0x1234)
 
-    @xfail
     def test_endian_int(self):
         if sys.byteorder == "little":
             self.assertIs(c_int.__ctype_le__, c_int)
@@ -80,7 +78,6 @@
         self.assertEqual(bin(s), "78563412")
         self.assertEqual(s.value, 0x12345678)
 
-    @xfail
     def test_endian_longlong(self):
         if sys.byteorder == "little":
             self.assertIs(c_longlong.__ctype_le__, c_longlong)
@@ -109,7 +106,6 @@
         self.assertEqual(bin(s), "EFCDAB9078563412")
         self.assertEqual(s.value, 0x1234567890ABCDEF)
 
-    @xfail
     def test_endian_float(self):
         if sys.byteorder == "little":
             self.assertIs(c_float.__ctype_le__, c_float)
@@ -128,7 +124,6 @@
         self.assertAlmostEqual(s.value, math.pi, 6)
         self.assertEqual(bin(struct.pack(">f", math.pi)), bin(s))
 
-    @xfail
     def test_endian_double(self):
         if sys.byteorder == "little":
             self.assertIs(c_double.__ctype_le__, c_double)
@@ -156,7 +151,6 @@
         self.assertIs(c_char.__ctype_le__, c_char)
         self.assertIs(c_char.__ctype_be__, c_char)
 
-    @xfail
     def test_struct_fields_1(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
@@ -192,7 +186,6 @@
                 pass
             self.assertRaises(TypeError, setattr, T, "_fields_", [("x", typ)])
 
-    @xfail
     def test_struct_struct(self):
         # nested structures with different byteorders
 
@@ -221,7 +214,6 @@
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
-    @xfail
     def test_struct_fields_2(self):
         # standard packing in struct uses no alignment.
         # So, we have to align using pad bytes.
@@ -245,7 +237,6 @@
         s2 = struct.pack(fmt, 0x12, 0x1234, 0x12345678, 3.14)
         self.assertEqual(bin(s1), bin(s2))
 
-    @xfail
     def test_unaligned_nonnative_struct_fields(self):
         if sys.byteorder == "little":
             base = BigEndianStructure
diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,10 +37,7 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            try:
-                o = typ()
-            except NotImplementedError as e:
-                self.skipTest(str(e))    # for PyPy
+            o = typ()
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/distutils/sysconfig_pypy.py b/lib-python/2.7/distutils/sysconfig_pypy.py
--- a/lib-python/2.7/distutils/sysconfig_pypy.py
+++ b/lib-python/2.7/distutils/sysconfig_pypy.py
@@ -218,6 +218,10 @@
 
         compiler.shared_lib_extension = so_ext
 
+def get_config_h_filename():
+    """Returns the path of pyconfig.h."""
+    inc_dir = get_python_inc(plat_specific=1)
+    return os.path.join(inc_dir, 'pyconfig.h')
 
 from sysconfig_cpython import (
     parse_makefile, _variable_rx, expand_makefile_vars)
diff --git a/lib-python/2.7/distutils/unixccompiler.py b/lib-python/2.7/distutils/unixccompiler.py
--- a/lib-python/2.7/distutils/unixccompiler.py
+++ b/lib-python/2.7/distutils/unixccompiler.py
@@ -226,7 +226,19 @@
         return "-L" + dir
 
     def _is_gcc(self, compiler_name):
-        return "gcc" in compiler_name or "g++" in compiler_name
+        # XXX PyPy workaround, look at the big comment below for more
+        # context. On CPython, the hack below works fine because
+        # `compiler_name` contains the name of the actual compiler which was
+        # used at compile time (e.g. 'x86_64-linux-gnu-gcc' on my machine).
+        # PyPy hardcodes it to 'cc', so the hack doesn't work, and the end
+        # result is that we pass the wrong option to the compiler.
+        #
+        # The workaround is to *always* pretend to be GCC if we are on Linux:
+        # this should cover the vast majority of real systems, including the
+        # ones which use clang (which understands the '-Wl,-rpath' syntax as
+        # well)
+        return (sys.platform == "linux2" or
+                "gcc" in compiler_name or "g++" in compiler_name)
 
     def runtime_library_dir_option(self, dir):
         # XXX Hackish, at the very least.  See Python bug #445902:
diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py
--- a/lib-python/2.7/inspect.py
+++ b/lib-python/2.7/inspect.py
@@ -203,7 +203,7 @@
         f_locals        local namespace seen by this frame
         f_restricted    0 or 1 if frame is in restricted execution mode
         f_trace         tracing function for this frame, or None"""
-    return isinstance(object, types.FrameType)
+    return isinstance(object, (types.FrameType, types.FakeFrameType))
 
 def iscode(object):
     """Return true if the object is a code object.
diff --git a/lib-python/2.7/multiprocessing/heap.py b/lib-python/2.7/multiprocessing/heap.py
--- a/lib-python/2.7/multiprocessing/heap.py
+++ b/lib-python/2.7/multiprocessing/heap.py
@@ -62,7 +62,7 @@
             self.size = size
             self.name = 'pym-%d-%d' % (os.getpid(), Arena._counter.next())
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == 0, 'tagname already in use'
+            #assert win32.GetLastError() == 0, 'tagname already in use'
             self._state = (self.size, self.name)
 
         def __getstate__(self):
@@ -72,7 +72,7 @@
         def __setstate__(self, state):
             self.size, self.name = self._state = state
             self.buffer = mmap.mmap(-1, self.size, tagname=self.name)
-            assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
+            #assert win32.GetLastError() == win32.ERROR_ALREADY_EXISTS
 
 else:
 
diff --git a/lib-python/2.7/string.py b/lib-python/2.7/string.py
--- a/lib-python/2.7/string.py
+++ b/lib-python/2.7/string.py
@@ -75,7 +75,7 @@
     for i in range(256):
         buf[i] = i
     for i in range(n):
-        buf[ord(fromstr[i])] = tostr[i]
+        buf[ord(fromstr[i])] = ord(tostr[i])
     return str(buf)
 
 
diff --git a/lib-python/2.7/types.py b/lib-python/2.7/types.py
--- a/lib-python/2.7/types.py
+++ b/lib-python/2.7/types.py
@@ -71,6 +71,12 @@
     FrameType = type(tb.tb_frame)
     del tb
 
+# PyPy extension
+try:
+    FakeFrameType = type(next(sys._current_frames().itervalues()))
+except (AttributeError, StopIteration):
+    FakeFrameType = FrameType
+
 SliceType = slice
 EllipsisType = type(Ellipsis)
 
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -82,7 +82,7 @@
         return False
 
     def in_dll(self, dll, name):
-        return self.from_address(dll._handle.getaddressindll(name))
+        return self.from_address(dll.__pypy_dll__.getaddressindll(name))
 
     def from_buffer(self, obj, offset=0):
         size = self._sizeofinstances()
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -430,7 +430,7 @@
             ffires = restype.get_ffi_argtype()
             return _ffi.FuncPtr.fromaddr(ptr, '', ffiargs, ffires, self._flags_)
 
-        cdll = self.dll._handle
+        cdll = self.dll.__pypy_dll__
         try:
             ffi_argtypes = [argtype.get_ffi_argtype() for argtype in argtypes]
             ffi_restype = restype.get_ffi_argtype()
diff --git a/lib_pypy/_ctypes/pointer.py b/lib_pypy/_ctypes/pointer.py
--- a/lib_pypy/_ctypes/pointer.py
+++ b/lib_pypy/_ctypes/pointer.py
@@ -142,6 +142,10 @@
         ptr._buffer = tp._ffiarray(1, autofree=True)
         ptr._buffer[0] = obj._buffer
         result = ptr
+    elif isinstance(obj, bytes):
+        result = tp()
+        result._buffer[0] = buffer(obj)._pypy_raw_address()
+        return result
     elif not (isinstance(obj, _CData) and type(obj)._is_pointer_like()):
         raise TypeError("cast() argument 1 must be a pointer, not %s"
                         % (type(obj),))
diff --git a/lib_pypy/_ctypes/primitive.py b/lib_pypy/_ctypes/primitive.py
--- a/lib_pypy/_ctypes/primitive.py
+++ b/lib_pypy/_ctypes/primitive.py
@@ -61,6 +61,54 @@
 
 pyobj_container = GlobalPyobjContainer()
 
+def swap_bytes(value, sizeof, typeof, get_or_set):
+    def swap_2():
+        return ((value >> 8) & 0x00FF) | ((value << 8) & 0xFF00)
+
+    def swap_4():
+        return ((value & 0x000000FF) << 24) | \
+               ((value & 0x0000FF00) << 8) | \
+               ((value & 0x00FF0000) >> 8) | \
+               ((value >> 24) & 0xFF)
+
+    def swap_8():
+        return ((value & 0x00000000000000FFL) << 56) | \
+               ((value & 0x000000000000FF00L) << 40) | \
+               ((value & 0x0000000000FF0000L) << 24) | \
+               ((value & 0x00000000FF000000L) << 8) | \
+               ((value & 0x000000FF00000000L) >> 8) | \
+               ((value & 0x0000FF0000000000L) >> 24) | \
+               ((value & 0x00FF000000000000L) >> 40) | \
+               ((value >> 56) & 0xFF)
+
+    def swap_double_float(typ):
+        from struct import pack, unpack
+        if get_or_set == 'set':
+            if sys.byteorder == 'little':
+                st = pack(''.join(['>', typ]), value)
+            else:
+                st = pack(''.join(['<', typ]), value)
+            return unpack(typ, st)[0]
+        else:
+            packed = pack(typ, value)
+            if sys.byteorder == 'little':
+                st = unpack(''.join(['>', typ]), packed)
+            else:
+                st = unpack(''.join(['<', typ]), packed)
+            return st[0]
+
+    if typeof in ('c_float', 'c_float_le', 'c_float_be'):
+        return swap_double_float('f')
+    elif typeof in ('c_double', 'c_double_le', 'c_double_be'):
+        return swap_double_float('d')
+    else:
+        if sizeof == 2:
+            return swap_2()
+        elif sizeof == 4:
+            return swap_4()
+        elif sizeof == 8:
+            return swap_8()
+
 def generic_xxx_p_from_param(cls, value):
     if value is None:
         return cls(None)
@@ -271,6 +319,31 @@
             def _as_ffi_pointer_(self, ffitype):
                 return as_ffi_pointer(self, ffitype)
             result._as_ffi_pointer_ = _as_ffi_pointer_
+        if name[-2:] != '_p' and name[-3:] not in ('_le', '_be') \
+                and name not in ('c_wchar', '_SimpleCData', 'c_longdouble', 'c_bool', 'py_object'):
+            from sys import byteorder
+            if byteorder == 'big':
+                name += '_le'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_le__ = swapped
+                result.__ctype_be__ = result
+                swapped.__ctype_be__ = result
+                swapped.__ctype_le__ = swapped
+            else:
+                name += '_be'
+                swapped = self.__new__(self, name, bases, dct)
+                result.__ctype_be__ = swapped
+                result.__ctype_le__ = result
+                swapped.__ctype_le__ = result
+                swapped.__ctype_be__ = swapped
+            from _ctypes import sizeof
+            def _getval(self):
+                return swap_bytes(self._buffer[0], sizeof(self), name, 'get')
+            def _setval(self, value):
+                d = result()
+                d.value = value
+                self._buffer[0] = swap_bytes(d.value, sizeof(self), name, 'set')
+            swapped.value = property(_getval, _setval)
 
         return result
 
diff --git a/lib_pypy/_ctypes/structure.py b/lib_pypy/_ctypes/structure.py
--- a/lib_pypy/_ctypes/structure.py
+++ b/lib_pypy/_ctypes/structure.py
@@ -40,6 +40,22 @@
         else:
             rawfields.append((f[0], f[1]._ffishape_))
 
+    # hack for duplicate field names
+    already_seen = set()
+    names1 = names
+    names = []
+    for f in names1:
+        if f not in already_seen:
+            names.append(f)
+            already_seen.add(f)
+    already_seen = set()
+    for i in reversed(range(len(rawfields))):
+        if rawfields[i][0] in already_seen:
+            rawfields[i] = (('$DUP%d$%s' % (i, rawfields[i][0]),)
+                            + rawfields[i][1:])
+        already_seen.add(rawfields[i][0])
+    # /hack
+
     _set_shape(self, rawfields, self._is_union)
 
     fields = {}
@@ -130,6 +146,7 @@
             obj._buffer.__setattr__(self.name, arg)
 
 
+
 def _set_shape(tp, rawfields, is_union=False):
     tp._ffistruct_ = _rawffi.Structure(rawfields, is_union,
                                       getattr(tp, '_pack_', 0))
@@ -224,19 +241,27 @@
         res.__dict__['_index'] = -1
         return res
 
-
 class StructOrUnion(_CData):
     __metaclass__ = StructOrUnionMeta
 
     def __new__(cls, *args, **kwds):
         from _ctypes import union
-        self = super(_CData, cls).__new__(cls)
-        if ('_abstract_' in cls.__dict__ or cls is Structure 
+        if ('_abstract_' in cls.__dict__ or cls is Structure
                                          or cls is union.Union):
             raise TypeError("abstract class")
         if hasattr(cls, '_swappedbytes_'):
-            raise NotImplementedError("missing in PyPy: structure/union with "
-                                      "swapped (non-native) byte ordering")
+            fields = [None] * len(cls._fields_)
+            for i in range(len(cls._fields_)):
+                if cls._fields_[i][1] == cls._fields_[i][1].__dict__.get('__ctype_be__', None):
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_le__', cls._fields_[i][1])
+                else:
+                    swapped = cls._fields_[i][1].__dict__.get('__ctype_be__', cls._fields_[i][1])
+                if len(cls._fields_[i]) < 3:
+                    fields[i] = (cls._fields_[i][0], swapped)
+                else:
+                    fields[i] = (cls._fields_[i][0], swapped, cls._fields_[i][2])
+            names_and_fields(cls, fields, _CData, cls.__dict__.get('_anonymous_', None))
+        self = super(_CData, cls).__new__(cls)
         if hasattr(cls, '_ffistruct_'):
             self.__dict__['_buffer'] = self._ffistruct_(autofree=True)
         return self
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -22,12 +22,27 @@
     linklibs = ['tcl', 'tk']
     libdirs = []
 else:
-    for _ver in ['', '8.6', '8.5', '']:
+    # On some Linux distributions, the tcl and tk libraries are
+    # stored in /usr/include, so we must check this case also
+    libdirs = []
+    found = False
+    for _ver in ['', '8.6', '8.5']:
         incdirs = ['/usr/include/tcl' + _ver]
         linklibs = ['tcl' + _ver, 'tk' + _ver]
-        libdirs = []
         if os.path.isdir(incdirs[0]):
+            found = True
             break
+    if not found:
+        for _ver in ['8.6', '8.5', '']:
+            incdirs = []
+            linklibs = ['tcl' + _ver, 'tk' + _ver]
+            if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
+                found = True
+                break
+    if not found:
+        sys.stderr.write("*** TCL libraries not found!  Falling back...\n")
+        incdirs = []
+        linklibs = ['tcl', 'tk']
 
 config_ffi = FFI()
 config_ffi.cdef("""
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -116,10 +116,20 @@
 
 @builtinify
 def dump(obj, file, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     Pickler(file, protocol).dump(obj)
 
 @builtinify
 def dumps(obj, protocol=None):
+    if protocol > HIGHEST_PROTOCOL:
+        # use cPickle error message, not pickle.py one
+        raise ValueError("pickle protocol %d asked for; "
+                     "the highest available protocol is %d" % (
+                     protocol, HIGHEST_PROTOCOL))
     file = StringIO()
     Pickler(file, protocol).dump(obj)
     return file.getvalue()
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.11.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.11.1"
+__version_info__ = (1, 11, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -95,6 +95,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -1,7 +1,12 @@
 
 /***** Support code for embedding *****/
 
-#if defined(_MSC_VER)
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+#if defined(_WIN32)
 #  define CFFI_DLLEXPORT  __declspec(dllexport)
 #elif defined(__GNUC__)
 #  define CFFI_DLLEXPORT  __attribute__((visibility("default")))
@@ -242,7 +247,7 @@
 
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.11.1"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
@@ -525,3 +530,7 @@
 #undef cffi_compare_and_swap
 #undef cffi_write_barrier
 #undef cffi_read_barrier
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -394,12 +394,17 @@
             replace_with = ' ' + replace_with
         return self._backend.getcname(cdecl, replace_with)
 
-    def gc(self, cdata, destructor):
+    def gc(self, cdata, destructor, size=0):
         """Return a new cdata object that points to the same
         data.  Later, when this new cdata object is garbage-collected,
         'destructor(old_cdata_object)' will be called.
+
+        The optional 'size' gives an estimate of the size, used to
+        trigger the garbage collection more eagerly.  So far only used
+        on PyPy.  It tells the GC that the returned object keeps alive
+        roughly 'size' bytes of external memory.
         """
-        return self._backend.gcp(cdata, destructor)
+        return self._backend.gcp(cdata, destructor, size)
 
     def _get_cached_btype(self, type):
         assert self._lock.acquire(False) is False
diff --git a/lib_pypy/cffi/backend_ctypes.py b/lib_pypy/cffi/backend_ctypes.py
--- a/lib_pypy/cffi/backend_ctypes.py
+++ b/lib_pypy/cffi/backend_ctypes.py
@@ -1002,7 +1002,7 @@
 
     _weakref_cache_ref = None
 
-    def gcp(self, cdata, destructor):
+    def gcp(self, cdata, destructor, size=0):
         if self._weakref_cache_ref is None:
             import weakref
             class MyRef(weakref.ref):
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -412,6 +412,9 @@
             prnt('    }')
         prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
+        prnt('#if PY_MAJOR_VERSION >= 3')
+        prnt('    return NULL;')
+        prnt('#endif')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
         # 'export_symbols', so instead of fighting it, just give up and
@@ -578,7 +581,7 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.BasePrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -296,7 +296,7 @@
 
     def _convert_expr_from_c(self, tp, var, context):
         if isinstance(tp, model.PrimitiveType):
-            if tp.is_integer_type():
+            if tp.is_integer_type() and tp.name != '_Bool':
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
@@ -872,6 +872,7 @@
 #define _cffi_from_c_ulong PyLong_FromUnsignedLong
 #define _cffi_from_c_longlong PyLong_FromLongLong
 #define _cffi_from_c_ulonglong PyLong_FromUnsignedLongLong
+#define _cffi_from_c__Bool PyBool_FromLong
 
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
diff --git a/lib_pypy/pyrepl/historical_reader.py b/lib_pypy/pyrepl/historical_reader.py
--- a/lib_pypy/pyrepl/historical_reader.py
+++ b/lib_pypy/pyrepl/historical_reader.py
@@ -17,7 +17,7 @@
 # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
 # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from pyrepl import reader, commands
+from pyrepl import reader, commands, input
 from pyrepl.reader import Reader as R
 
 isearch_keymap = tuple(
@@ -214,7 +214,6 @@
                   isearch_forwards, isearch_backwards, operate_and_get_next]:
             self.commands[c.__name__] = c
             self.commands[c.__name__.replace('_', '-')] = c
-        from pyrepl import input
         self.isearch_trans = input.KeymapTranslator(
             isearch_keymap, invalid_cls=isearch_end,
             character_cls=isearch_add_character)
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -36,7 +36,7 @@
     "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array",
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend",
-    "_csv", "cppyy", "_pypyjson", "_jitlog"
+    "_csv", "_cppyy", "_pypyjson", "_jitlog"
 ])
 
 from rpython.jit.backend import detect_cpu
@@ -67,10 +67,12 @@
         if name in translation_modules:
             translation_modules.remove(name)
 
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # not tested on win32
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # not tested on win32
     if "faulthandler" in working_modules:
         working_modules.remove("faulthandler")  # missing details
+    if "_vmprof" in working_modules:
+        working_modules.remove("_vmprof")  # FIXME: missing details
 
     # The _locale module is needed by site.py on Windows
     default_modules.add("_locale")
@@ -79,8 +81,8 @@
     working_modules.remove('fcntl')  # LOCK_NB not defined
     working_modules.remove("_minimal_curses")
     working_modules.remove("termios")
-    if "cppyy" in working_modules:
-        working_modules.remove("cppyy")  # depends on ctypes
+    if "_cppyy" in working_modules:
+        working_modules.remove("_cppyy")  # depends on ctypes
 
 #if sys.platform.startswith("linux"):
 #    _mach = os.popen('uname -m', 'r').read().strip()
@@ -92,7 +94,7 @@
     '_multiprocessing': [('objspace.usemodules.time', True),
                          ('objspace.usemodules.thread', True)],
     'cpyext': [('objspace.usemodules.array', True)],
-    'cppyy': [('objspace.usemodules.cpyext', True)],
+    '_cppyy': [('objspace.usemodules.cpyext', True)],
     'faulthandler': [('objspace.usemodules._vmprof', True)],
     }
 module_suggests = {
@@ -224,11 +226,6 @@
                    "use specialised tuples",
                    default=False),
 
-        BoolOption("withcelldict",
-                   "use dictionaries that are optimized for being used as module dicts",
-                   default=False,
-                   requires=[("objspace.honor__builtins__", False)]),
-
         BoolOption("withliststrategies",
                    "enable optimized ways to store lists of primitives ",
                    default=True),
@@ -288,7 +285,7 @@
 
     # extra optimizations with the JIT
     if level == 'jit':
-        config.objspace.std.suggest(withcelldict=True)
+        pass # none at the moment
 
 
 def enable_allworkingmodules(config):
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -10,6 +10,18 @@
 minutes on a fast machine -- and RAM-hungry.  You will need **at least** 2 GB
 of memory on a 32-bit machine and 4GB on a 64-bit machine.
 
+Before you start
+----------------
+
+Our normal development workflow avoids a full translation by using test-driven
+development. You can read more about how to develop PyPy here_, and latest
+translated (hopefully functional) binary packages are available on our
+buildbot's `nightly builds`_
+
+.. _here: getting-started-dev.html
+.. _`nightly builds`: http://buildbot.pypy.org/nightly
+
+You will need the build dependencies below to run the tests.
 
 Clone the repository
 --------------------
@@ -107,8 +119,15 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian and Ubuntu, this is the command to install all build-time
-dependencies::
+On recent Debian and Ubuntu (like 17.04), this is the command to install
+all build-time dependencies::
+
+    apt-get install gcc make libffi-dev pkg-config zlib1g-dev libbz2-dev \
+    libsqlite3-dev libncurses5-dev libexpat1-dev libssl-dev libgdbm-dev \
+    tk-dev libgc-dev python-cffi \
+    liblzma-dev libncursesw5-dev     # these two only needed on PyPy3
+
+On older Debian and Ubuntu (12.04 to 16.04)::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
@@ -140,22 +159,61 @@
 Run the translation
 -------------------
 
+We usually translate in the ``pypy/goal`` directory, so all the following
+commands assume your ``$pwd`` is there.
+
 Translate with JIT::
 
-    cd pypy/goal
     pypy ../../rpython/bin/rpython --opt=jit
 
 Translate without JIT::
 
-    cd pypy/goal
     pypy ../../rpython/bin/rpython --opt=2
 
+Note this translates pypy via the ``targetpypystandalone.py`` file, so these
+are shorthand for::
+
+    pypy ../../rpython/bin/rpython <rpython options> targetpypystandalone.py <pypy options>
+
+More help is availabe via ``--help`` at either option position, and more info
+can be found in the :doc:`config/index` section.
+
 (You can use ``python`` instead of ``pypy`` here, which will take longer
 but works too.)
 
-If everything works correctly this will create an executable ``pypy-c`` in the
-current directory. The executable behaves mostly like a normal Python
-interpreter (see :doc:`cpython_differences`).
+If everything works correctly this will:
+
+1. Run the rpython `translation chain`_, producing a database of the
+   entire pypy interpreter. This step is currently singe threaded, and RAM
+   hungry. As part of this step,  the chain creates a large number of C code
+   files and a Makefile to compile them in a
+   directory controlled by the ``PYPY_USESSION_DIR`` environment variable.  
+2. Create an executable ``pypy-c`` by running the Makefile. This step can
+   utilize all possible cores on the machine.  
+3. Copy the needed binaries to the current directory.  
+4. Generate c-extension modules for any cffi-based stdlib modules.  
+
+
+The resulting executable behaves mostly like a normal Python
+interpreter (see :doc:`cpython_differences`), and is ready for testing, for
+use as a base interpreter for a new virtualenv, or for packaging into a binary
+suitable for installation on another machine running the same OS as the build
+machine. 
+
+Note that step 4 is merely done as a convenience, any of the steps may be rerun
+without rerunning the previous steps.
+
+.. _`translation chain`: https://rpython.readthedocs.io/en/latest/translation.html
+
+
+Making a debug build of PyPy
+----------------------------
+
+If the Makefile is rerun with the lldebug or lldebug0 target, appropriate
+compilation flags are added to add debug info and reduce compiler optimizations
+to ``-O0`` respectively. If you stop in a debugger, you will see the
+very wordy machine-generated C code from the rpython translation step, which
+takes a little bit of reading to relate back to the rpython code.
 
 Build cffi import libraries for the stdlib
 ------------------------------------------
@@ -169,14 +227,6 @@
 
 .. _`out-of-line API mode`: http://cffi.readthedocs.org/en/latest/overview.html#real-example-api-level-out-of-line
 
-Translating with non-standard options
--------------------------------------
-
-It is possible to have non-standard features enabled for translation,
-but they are not really tested any more.  Look, for example, at the
-:doc:`objspace proxies <objspace-proxies>` document.
-
-
 Packaging (preparing for installation)
 --------------------------------------
 
@@ -205,14 +255,16 @@
 
 * PyPy 2.5.1 or earlier: normal users would see permission errors.
   Installers need to run ``pypy -c "import gdbm"`` and other similar
-  commands at install time; the exact list is in `package.py`_.  Users
+  commands at install time; the exact list is in 
+  :source:`pypy/tool/release/package.py <package.py>`.  Users
   seeing a broken installation of PyPy can fix it after-the-fact if they
   have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
 
 * PyPy 2.6 and later: anyone would get ``ImportError: no module named
   _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
   ``lib_pypy`` directory during the installation process (plus others;
-  see the exact list in `package.py`_).  Users seeing a broken
+  see the exact list in :source:`pypy/tool/release/package.py <package.py>`).
+  Users seeing a broken
   installation of PyPy can fix it after-the-fact, by running ``pypy
   /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
   called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
diff --git a/pypy/doc/config/objspace.std.withcelldict.txt b/pypy/doc/config/objspace.std.withcelldict.txt
deleted file mode 100644
--- a/pypy/doc/config/objspace.std.withcelldict.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-Enable cell-dicts. This optimization is not helpful without the JIT. In the
-presence of the JIT, it greatly helps looking up globals.
diff --git a/pypy/doc/configuration.rst b/pypy/doc/configuration.rst
--- a/pypy/doc/configuration.rst
+++ b/pypy/doc/configuration.rst
@@ -188,4 +188,6 @@
 can be found on the ``config`` attribute of all ``TranslationContext``
 instances and are described in :source:`rpython/config/translationoption.py`. The interpreter options
 are attached to the object space, also under the name ``config`` and are
-described in :source:`pypy/config/pypyoption.py`.
+described in :source:`pypy/config/pypyoption.py`. Both set of options are
+documented in the :doc:`config/index` section.
+
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -27,8 +27,8 @@
   Wim Lavrijsen
   Eric van Riet Paap
   Richard Emslie
+  Remi Meier
   Alexander Schremmer
-  Remi Meier
   Dan Villiom Podlaski Christiansen
   Lukas Diekmann
   Sven Hager
@@ -69,6 +69,7 @@
   Michael Foord
   Stephan Diehl
   Stefano Rivera
+  Jean-Paul Calderone
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
@@ -77,14 +78,13 @@
   Bob Ippolito
   Bruno Gola
   David Malcolm
-  Jean-Paul Calderone
   Squeaky
   Edd Barrett
   Timo Paulssen
   Marius Gedminas
+  Nicolas Truessel
   Alexandre Fayolle
   Simon Burton
-  Nicolas Truessel
   Martin Matusiak
   Laurence Tratt
   Wenzhu Man
@@ -123,6 +123,7 @@
   Stefan H. Muller
   Tim Felgentreff
   Eugene Oden
+  Dodan Mihai
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -149,11 +150,13 @@
   Rocco Moretti
   Gintautas Miliauskas
   Lucian Branescu Mihaila
+  Mariano Anaya
   anatoly techtonik
-  Dodan Mihai
   Karl Bartel
+  Stefan Beyer
   Gabriel Lavoie
   Jared Grubb
+  Alecsandru Patrascu
   Olivier Dormond
   Wouter van Heyst
   Sebastian Pawlu&#347;
@@ -161,6 +164,7 @@
   Victor Stinner
   Andrews Medina
   Aaron Iles
+  p_zieschang at yahoo.de
   Toby Watson
   Daniel Patrick
   Stuart Williams
@@ -171,6 +175,7 @@
   Michael Cheng
   Mikael Sch&#246;nenberg
   Stanislaw Halik
+  Mihnea Saracin
   Berkin Ilbeyi
   Gasper Zejn
   Faye Zhao
@@ -181,14 +186,12 @@
   Jonathan David Riehl
   Beatrice During
   Alex Perry
-  p_zieschang at yahoo.de
   Robert Zaremba
   Alan McIntyre
   Alexander Sedov
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
-  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
@@ -210,7 +213,6 @@
   Jacek Generowicz
   Sylvain Thenault
   Jakub Stasiak
-  Stefan Beyer
   Andrew Dalke
   Alejandro J. Cura
   Vladimir Kryachko
@@ -242,6 +244,7 @@
   Christoph Gerum
   Miguel de Val Borro
   Artur Lisiecki
+  afteryu
   Toni Mattis
   Laurens Van Houtven
   Bobby Impollonia
@@ -272,6 +275,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Anthony Sottile
   Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
@@ -292,12 +296,14 @@
   Mike Bayer
   Rodrigo Ara&#250;jo
   Daniil Yarancev
+  Min RK
   OlivierBlanvillain
   Jonas Pfannschmidt
   Zearin
   Andrey Churin
   Dan Crosta
   reubano at gmail.com
+  Stanis&#322;aw Halik
   Julien Phalip
   Roman Podoliaka
   Eli Stevens
diff --git a/pypy/doc/cppyy.rst b/pypy/doc/cppyy.rst
deleted file mode 100644
--- a/pypy/doc/cppyy.rst
+++ /dev/null
@@ -1,672 +0,0 @@
-cppyy: C++ bindings for PyPy
-============================
-
-The cppyy module delivers dynamic Python-C++ bindings.
-It is designed for automation, high performance, scale, interactivity, and
-handling all of modern C++ (11, 14, etc.).
-It is based on `Cling`_ which, through `LLVM`_/`clang`_, provides C++
-reflection and interactivity.
-Reflection information is extracted from C++ header files.
-Cppyy itself is built into PyPy (an alternative exists for CPython), but
-it requires a `backend`_, installable through pip, to interface with Cling.
-
-.. _Cling: https://root.cern.ch/cling
-.. _LLVM: http://llvm.org/
-.. _clang: http://clang.llvm.org/
-.. _backend: https://pypi.python.org/pypi/PyPy-cppyy-backend
-
-
-Installation
-------------
-
-This assumes PyPy2.7 v5.7 or later; earlier versions use a Reflex-based cppyy
-module, which is no longer supported.
-Both the tooling and user-facing Python codes are very backwards compatible,
-however.
-Further dependencies are cmake (for general build), Python2.7 (for LLVM), and
-a modern C++ compiler (one that supports at least C++11).
-
-Assuming you have a recent enough version of PyPy installed, use pip to
-complete the installation of cppyy::
-
- $ MAKE_NPROCS=4 pypy-c -m pip install --verbose PyPy-cppyy-backend
-
-Set the number of parallel builds ('4' in this example, through the MAKE_NPROCS
-environment variable) to a number appropriate for your machine.
-The building process may take quite some time as it includes a customized
-version of LLVM as part of Cling, which is why --verbose is recommended so that
-you can see the build progress.
-
-The default installation will be under
-$PYTHONHOME/site-packages/cppyy_backend/lib,
-which needs to be added to your dynamic loader path (LD_LIBRARY_PATH).
-If you need the dictionary and class map generation tools (used in the examples
-below), you need to add $PYTHONHOME/site-packages/cppyy_backend/bin to your
-executable path (PATH).
-
-
-Basic bindings example
-----------------------
-
-These examples assume that cppyy_backend is pointed to by the environment
-variable CPPYYHOME, and that CPPYYHOME/lib is added to LD_LIBRARY_PATH and
-CPPYYHOME/bin to PATH.
-
-Let's first test with a trivial example whether all packages are properly
-installed and functional.
-Create a C++ header file with some class in it (all functions are made inline
-for convenience; if you have out-of-line code, link with it as appropriate)::
-
-    $ cat MyClass.h
-    class MyClass {
-    public:
-        MyClass(int i = -99) : m_myint(i) {}
-
-        int GetMyInt() { return m_myint; }
-        void SetMyInt(int i) { m_myint = i; }
-
-    public:
-        int m_myint;
-    };
-
-Then, generate the bindings using ``genreflex`` (installed under
-cppyy_backend/bin in site_packages), and compile the code::
-
-    $ genreflex MyClass.h
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$CPPYYHOME/lib -lCling
-
-Next, make sure that the library can be found through the dynamic lookup path
-(the ``LD_LIBRARY_PATH`` environment variable on Linux, ``PATH`` on Windows),
-for example by adding ".".
-Now you're ready to use the bindings.
-Since the bindings are designed to look pythonistic, it should be
-straightforward::
-
-    $ pypy-c
-    >>>> import cppyy
-    >>>> cppyy.load_reflection_info("libMyClassDict.so")
-    <CPPLibrary object at 0xb6fd7c4c>
-    >>>> myinst = cppyy.gbl.MyClass(42)
-    >>>> print myinst.GetMyInt()
-    42
-    >>>> myinst.SetMyInt(33)
-    >>>> print myinst.m_myint
-    33
-    >>>> myinst.m_myint = 77
-    >>>> print myinst.GetMyInt()
-    77
-    >>>> help(cppyy.gbl.MyClass)   # shows that normal python introspection works
-
-That's all there is to it!
-
-
-Automatic class loader
-----------------------
-
-There is one big problem in the code above, that prevents its use in a (large
-scale) production setting: the explicit loading of the reflection library.
-Clearly, if explicit load statements such as these show up in code downstream
-from the ``MyClass`` package, then that prevents the ``MyClass`` author from
-repackaging or even simply renaming the dictionary library.
-
-The solution is to make use of an automatic class loader, so that downstream
-code never has to call ``load_reflection_info()`` directly.
-The class loader makes use of so-called rootmap files, which ``genreflex``
-can produce.
-These files contain the list of available C++ classes and specify the library
-that needs to be loaded for their use (as an aside, this listing allows for a
-cross-check to see whether reflection info is generated for all classes that
-you expect).
-By convention, the rootmap files should be located next to the reflection info
-libraries, so that they can be found through the normal shared library search
-path.
-They can be concatenated together, or consist of a single rootmap file per
-library.
-For example::
-
-    $ genreflex MyClass.h --rootmap=libMyClassDict.rootmap --rootmap-lib=libMyClassDict.so
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include MyClass_rflx.cpp -o libMyClassDict.so -L$CPPYYHOME/lib -lCling
-
-where the first option (``--rootmap``) specifies the output file name, and the
-second option (``--rootmap-lib``) the name of the reflection library where
-``MyClass`` will live.
-It is necessary to provide that name explicitly, since it is only in the
-separate linking step where this name is fixed.
-If the second option is not given, the library is assumed to be libMyClass.so,
-a name that is derived from the name of the header file.
-
-With the rootmap file in place, the above example can be rerun without explicit
-loading of the reflection info library::
-
-    $ pypy-c
-    >>>> import cppyy
-    >>>> myinst = cppyy.gbl.MyClass(42)
-    >>>> print myinst.GetMyInt()
-    42
-    >>>> # etc. ...
-
-As a caveat, note that the class loader is currently limited to classes only.
-
-
-Advanced example
-----------------
-
-The following snippet of C++ is very contrived, to allow showing that such
-pathological code can be handled and to show how certain features play out in
-practice::
-
-    $ cat MyAdvanced.h
-    #include <string>
-
-    class Base1 {
-    public:
-        Base1(int i) : m_i(i) {}
-        virtual ~Base1() {}
-        int m_i;
-    };
-
-    class Base2 {
-    public:
-        Base2(double d) : m_d(d) {}
-        virtual ~Base2() {}
-        double m_d;
-    };
-
-    class C;
-
-    class Derived : public virtual Base1, public virtual Base2 {
-    public:
-        Derived(const std::string& name, int i, double d) : Base1(i), Base2(d), m_name(name) {}
-        virtual C* gimeC() { return (C*)0; }
-        std::string m_name;
-    };
-
-    Base2* BaseFactory(const std::string& name, int i, double d) {
-        return new Derived(name, i, d);
-    }
-
-This code is still only in a header file, with all functions inline, for
-convenience of the example.
-If the implementations live in a separate source file or shared library, the
-only change needed is to link those in when building the reflection library.
-
-If you were to run ``genreflex`` like above in the basic example, you will
-find that not all classes of interest will be reflected, nor will be the
-global factory function.
-In particular, ``std::string`` will be missing, since it is not defined in
-this header file, but in a header file that is included.
-In practical terms, general classes such as ``std::string`` should live in a
-core reflection set, but for the moment assume we want to have it in the
-reflection library that we are building for this example.
-
-The ``genreflex`` script can be steered using a so-called `selection file`_
-(see "Generating Reflex Dictionaries")
-which is a simple XML file specifying, either explicitly or by using a
-pattern, which classes, variables, namespaces, etc. to select from the given
-header file.
-With the aid of a selection file, a large project can be easily managed:
-simply ``#include`` all relevant headers into a single header file that is
-handed to ``genreflex``.
-In fact, if you hand multiple header files to ``genreflex``, then a selection
-file is almost obligatory: without it, only classes from the last header will
-be selected.
-Then, apply a selection file to pick up all the relevant classes.
-For our purposes, the following rather straightforward selection will do
-(the name ``lcgdict`` for the root is historical, but required)::
-
-    $ cat MyAdvanced.xml
-    <lcgdict>
-        <class pattern="Base?" />
-        <class name="Derived" />
-        <class name="std::string" />
-        <function name="BaseFactory" />
-    </lcgdict>
-
-.. _selection file: https://root.cern.ch/how/how-use-reflex
-
-Now the reflection info can be generated and compiled::
-
-    $ genreflex MyAdvanced.h --selection=MyAdvanced.xml
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include MyAdvanced_rflx.cpp -o libAdvExDict.so -L$CPPYYHOME/lib -lCling
-
-and subsequently be used from PyPy::
-
-    >>>> import cppyy
-    >>>> cppyy.load_reflection_info("libAdvExDict.so")
-    <CPPLibrary object at 0x00007fdb48fc8120>
-    >>>> d = cppyy.gbl.BaseFactory("name", 42, 3.14)
-    >>>> type(d)
-    <class '__main__.Derived'>
-    >>>> isinstance(d, cppyy.gbl.Base1)
-    True
-    >>>> isinstance(d, cppyy.gbl.Base2)
-    True
-    >>>> d.m_i, d.m_d
-    (42, 3.14)
-    >>>> d.m_name == "name"
-    True
-    >>>>
-
-Again, that's all there is to it!
-
-A couple of things to note, though.
-If you look back at the C++ definition of the ``BaseFactory`` function,
-you will see that it declares the return type to be a ``Base2``, yet the
-bindings return an object of the actual type ``Derived``?
-This choice is made for a couple of reasons.
-First, it makes method dispatching easier: if bound objects are always their
-most derived type, then it is easy to calculate any offsets, if necessary.
-Second, it makes memory management easier: the combination of the type and
-the memory address uniquely identifies an object.
-That way, it can be recycled and object identity can be maintained if it is
-entered as a function argument into C++ and comes back to PyPy as a return
-value.
-Last, but not least, casting is decidedly unpythonistic.
-By always providing the most derived type known, casting becomes unnecessary.
-For example, the data member of ``Base2`` is simply directly available.
-Note also that the unreflected ``gimeC`` method of ``Derived`` does not
-preclude its use.
-It is only the ``gimeC`` method that is unusable as long as class ``C`` is
-unknown to the system.
-
-
-Features
---------
-
-The following is not meant to be an exhaustive list, since cppyy is still
-under active development.
-Furthermore, the intention is that every feature is as natural as possible on
-the python side, so if you find something missing in the list below, simply
-try it out.
-It is not always possible to provide exact mapping between python and C++
-(active memory management is one such case), but by and large, if the use of a
-feature does not strike you as obvious, it is more likely to simply be a bug.
-That is a strong statement to make, but also a worthy goal.
-For the C++ side of the examples, refer to this :doc:`example code <cppyy_example>`, which was
-bound using::
-
-    $ genreflex example.h --deep --rootmap=libexampleDict.rootmap --rootmap-lib=libexampleDict.so
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include example_rflx.cpp -o libexampleDict.so -L$CPPYYHOME/lib -lCling
-
-* **abstract classes**: Are represented as python classes, since they are
-  needed to complete the inheritance hierarchies, but will raise an exception
-  if an attempt is made to instantiate from them.
-  Example::
-
-    >>>> from cppyy.gbl import AbstractClass, ConcreteClass
-    >>>> a = AbstractClass()
-    Traceback (most recent call last):
-      File "<console>", line 1, in <module>
-    TypeError: cannot instantiate abstract class 'AbstractClass'
-    >>>> issubclass(ConcreteClass, AbstractClass)
-    True
-    >>>> c = ConcreteClass()
-    >>>> isinstance(c, AbstractClass)
-    True
-    >>>>
-
-* **arrays**: Supported for builtin data types only, as used from module
-  ``array``.
-  Out-of-bounds checking is limited to those cases where the size is known at
-  compile time (and hence part of the reflection info).
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> from array import array
-    >>>> c = ConcreteClass()
-    >>>> c.array_method(array('d', [1., 2., 3., 4.]), 4)
-    1 2 3 4
-    >>>>
-
-* **builtin data types**: Map onto the expected equivalent python types, with
-  the caveat that there may be size differences, and thus it is possible that
-  exceptions are raised if an overflow is detected.
-
-* **casting**: Is supposed to be unnecessary.
-  Object pointer returns from functions provide the most derived class known
-  in the hierarchy of the object being returned.
-  This is important to preserve object identity as well as to make casting,
-  a pure C++ feature after all, superfluous.
-  Example::
-
-    >>>> from cppyy.gbl import AbstractClass, ConcreteClass
-    >>>> c = ConcreteClass()
-    >>>> ConcreteClass.show_autocast.__doc__
-    'AbstractClass* ConcreteClass::show_autocast()'
-    >>>> d = c.show_autocast()
-    >>>> type(d)
-    <class '__main__.ConcreteClass'>
-    >>>>
-
-  However, if need be, you can perform C++-style reinterpret_casts (i.e.
-  without taking offsets into account), by taking and rebinding the address
-  of an object::
-
-    >>>> from cppyy import addressof, bind_object
-    >>>> e = bind_object(addressof(d), AbstractClass)
-    >>>> type(e)
-    <class '__main__.AbstractClass'>
-    >>>>
-
-* **classes and structs**: Get mapped onto python classes, where they can be
-  instantiated as expected.
-  If classes are inner classes or live in a namespace, their naming and
-  location will reflect that.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass, Namespace
-    >>>> ConcreteClass == Namespace.ConcreteClass
-    False
-    >>>> n = Namespace.ConcreteClass.NestedClass()
-    >>>> type(n)
-    <class '__main__.Namespace::ConcreteClass::NestedClass'>
-    >>>>
-
-* **data members**: Public data members are represented as python properties
-  and provide read and write access on instances as expected.
-  Private and protected data members are not accessible.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> c = ConcreteClass()
-    >>>> c.m_int
-    42
-    >>>>
-
-* **default arguments**: C++ default arguments work as expected, but python
-  keywords are not supported.
-  It is technically possible to support keywords, but for the C++ interface,
-  the formal argument names have no meaning and are not considered part of the
-  API, hence it is not a good idea to use keywords.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> c = ConcreteClass()       # uses default argument
-    >>>> c.m_int
-    42
-    >>>> c = ConcreteClass(13)
-    >>>> c.m_int
-    13
-    >>>>
-
-* **doc strings**: The doc string of a method or function contains the C++
-  arguments and return types of all overloads of that name, as applicable.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> print ConcreteClass.array_method.__doc__
-    void ConcreteClass::array_method(int*, int)
-    void ConcreteClass::array_method(double*, int)
-    >>>>
-
-* **enums**: Are translated as ints with no further checking.
-
-* **functions**: Work as expected and live in their appropriate namespace
-  (which can be the global one, ``cppyy.gbl``).
-
-* **inheritance**: All combinations of inheritance on the C++ (single,
-  multiple, virtual) are supported in the binding.
-  However, new python classes can only use single inheritance from a bound C++
-  class.
-  Multiple inheritance would introduce two "this" pointers in the binding.
-  This is a current, not a fundamental, limitation.
-  The C++ side will not see any overridden methods on the python side, as
-  cross-inheritance is planned but not yet supported.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> help(ConcreteClass)
-    Help on class ConcreteClass in module __main__:
-
-    class ConcreteClass(AbstractClass)
-     |  Method resolution order:
-     |      ConcreteClass
-     |      AbstractClass
-     |      cppyy.CPPObject
-     |      __builtin__.CPPInstance
-     |      __builtin__.object
-     |
-     |  Methods defined here:
-     |
-     |  ConcreteClass(self, *args)
-     |      ConcreteClass::ConcreteClass(const ConcreteClass&)
-     |      ConcreteClass::ConcreteClass(int)
-     |      ConcreteClass::ConcreteClass()
-     |
-     etc. ....
-
-* **memory**: C++ instances created by calling their constructor from python
-  are owned by python.
-  You can check/change the ownership with the _python_owns flag that every
-  bound instance carries.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> c = ConcreteClass()
-    >>>> c._python_owns            # True: object created in Python
-    True
-    >>>>
-
-* **methods**: Are represented as python methods and work as expected.
-  They are first class objects and can be bound to an instance.
-  Virtual C++ methods work as expected.
-  To select a specific virtual method, do like with normal python classes
-  that override methods: select it from the class that you need, rather than
-  calling the method on the instance.
-  To select a specific overload, use the __dispatch__ special function, which
-  takes the name of the desired method and its signature (which can be
-  obtained from the doc string) as arguments.
-
-* **namespaces**: Are represented as python classes.
-  Namespaces are more open-ended than classes, so sometimes initial access may
-  result in updates as data and functions are looked up and constructed
-  lazily.
-  Thus the result of ``dir()`` on a namespace shows the classes available,
-  even if they may not have been created yet.
-  It does not show classes that could potentially be loaded by the class
-  loader.
-  Once created, namespaces are registered as modules, to allow importing from
-  them.
-  Namespace currently do not work with the class loader.
-  Fixing these bootstrap problems is on the TODO list.
-  The global namespace is ``cppyy.gbl``.
-
-* **NULL**: Is represented as ``cppyy.gbl.nullptr``.
-  In C++11, the keyword ``nullptr`` is used to represent ``NULL``.
-  For clarity of intent, it is recommended to use this instead of ``None``
-  (or the integer ``0``, which can serve in some cases), as ``None`` is better
-  understood as ``void`` in C++.
-
-* **operator conversions**: If defined in the C++ class and a python
-  equivalent exists (i.e. all builtin integer and floating point types, as well
-  as ``bool``), it will map onto that python conversion.
-  Note that ``char*`` is mapped onto ``__str__``.
-  Example::
-
-    >>>> from cppyy.gbl import ConcreteClass
-    >>>> print ConcreteClass()
-    Hello operator const char*!
-    >>>>
-
-* **operator overloads**: If defined in the C++ class and if a python
-  equivalent is available (not always the case, think e.g. of ``operator||``),
-  then they work as expected.
-  Special care needs to be taken for global operator overloads in C++: first,
-  make sure that they are actually reflected, especially for the global
-  overloads for ``operator==`` and ``operator!=`` of STL vector iterators in
-  the case of gcc (note that they are not needed to iterate over a vector).
-  Second, make sure that reflection info is loaded in the proper order.
-  I.e. that these global overloads are available before use.
-
-* **pointers**: For builtin data types, see arrays.
-  For objects, a pointer to an object and an object looks the same, unless
-  the pointer is a data member.
-  In that case, assigning to the data member will cause a copy of the pointer
-  and care should be taken about the object's life time.
-  If a pointer is a global variable, the C++ side can replace the underlying
-  object and the python side will immediately reflect that.
-
-* **PyObject***: Arguments and return types of ``PyObject*`` can be used, and
-  passed on to CPython API calls.
-  Since these CPython-like objects need to be created and tracked (this all
-  happens through ``cpyext``) this interface is not particularly fast.
-
-* **static data members**: Are represented as python property objects on the
-  class and the meta-class.
-  Both read and write access is as expected.
-
-* **static methods**: Are represented as python's ``staticmethod`` objects
-  and can be called both from the class as well as from instances.
-
-* **strings**: The std::string class is considered a builtin C++ type and
-  mixes quite well with python's str.
-  Python's str can be passed where a ``const char*`` is expected, and an str
-  will be returned if the return type is ``const char*``.
-
-* **templated classes**: Are represented in a meta-class style in python.
-  This may look a little bit confusing, but conceptually is rather natural.
-  For example, given the class ``std::vector<int>``, the meta-class part would
-  be ``std.vector``.
-  Then, to get the instantiation on ``int``, do ``std.vector(int)`` and to
-  create an instance of that class, do ``std.vector(int)()``::
-
-    >>>> import cppyy
-    >>>> cppyy.load_reflection_info('libexampleDict.so')
-    >>>> cppyy.gbl.std.vector                # template metatype
-    <cppyy.CppyyTemplateType object at 0x00007fcdd330f1a0>
-    >>>> cppyy.gbl.std.vector(int)           # instantiates template -> class
-    <class '__main__.std::vector<int>'>
-    >>>> cppyy.gbl.std.vector(int)()         # instantiates class -> object
-    <__main__.std::vector<int> object at 0x00007fe480ba4bc0>
-    >>>>
-
-  Note that templates can be build up by handing actual types to the class
-  instantiation (as done in this vector example), or by passing in the list of
-  template arguments as a string.
-  The former is a lot easier to work with if you have template instantiations
-  using classes that themselves are templates in  the arguments (think e.g a
-  vector of vectors).
-  All template classes must already exist in the loaded reflection info, they
-  do not work (yet) with the class loader.
-
-  For compatibility with other bindings generators, use of square brackets
-  instead of parenthesis to instantiate templates is supported as well.
-
-* **templated functions**: Automatically participate in overloading and are
-  used in the same way as other global functions.
-
-* **templated methods**: For now, require an explicit selection of the
-  template parameters.
-  This will be changed to allow them to participate in overloads as expected.
-
-* **typedefs**: Are simple python references to the actual classes to which
-  they refer.
-
-* **unary operators**: Are supported if a python equivalent exists, and if the
-  operator is defined in the C++ class.
-
-You can always find more detailed examples and see the full of supported
-features by looking at the tests in pypy/module/cppyy/test.
-
-If a feature or reflection info is missing, this is supposed to be handled
-gracefully.
-In fact, there are unit tests explicitly for this purpose (even as their use
-becomes less interesting over time, as the number of missing features
-decreases).
-Only when a missing feature is used, should there be an exception.
-For example, if no reflection info is available for a return type, then a
-class that has a method with that return type can still be used.
-Only that one specific method can not be used.
-
-
-Templates
----------
-
-Templates can be automatically instantiated, assuming the appropriate header
-files have been loaded or are accessible to the class loader.
-This is the case for example for all of STL.
-For example::
-
-    $ cat MyTemplate.h
-    #include <vector>
-
-    class MyClass {
-    public:
-        MyClass(int i = -99) : m_i(i) {}
-        MyClass(const MyClass& s) : m_i(s.m_i) {}
-        MyClass& operator=(const MyClass& s) { m_i = s.m_i; return *this; }
-        ~MyClass() {}
-        int m_i;
-    };
-
-Run the normal ``genreflex`` and compilation steps::
-
-    $ genreflex MyTemplate.h --selection=MyTemplate.xml
-    $ g++ -std=c++11 -fPIC -rdynamic -O2 -shared -I$CPPYYHOME/include MyTemplate_rflx.cpp -o libTemplateDict.so -L$CPPYYHOME/lib -lCling
-
-Subsequent use should be as expected.
-Note the meta-class style of "instantiating" the template::
-
-    >>>> import cppyy
-    >>>> cppyy.load_reflection_info("libTemplateDict.so")
-    >>>> std = cppyy.gbl.std
-    >>>> MyClass = cppyy.gbl.MyClass
-    >>>> v = std.vector(MyClass)()
-    >>>> v += [MyClass(1), MyClass(2), MyClass(3)]
-    >>>> for m in v:
-    ....     print m.m_i,
-    ....
-    1 2 3
-    >>>>
-
-The arguments to the template instantiation can either be a string with the
-full list of arguments, or the explicit classes.
-The latter makes for easier code writing if the classes passed to the
-instantiation are themselves templates.
-
-
-The fast lane
--------------
-
-By default, cppyy will use direct function pointers through `CFFI`_ whenever
-possible. If this causes problems for you, you can disable it by setting the
-CPPYY_DISABLE_FASTPATH environment variable.
-
-.. _CFFI: https://cffi.readthedocs.io/en/latest/
-
-
-CPython
--------
-
-Most of the ideas in cppyy come originally from the `PyROOT`_ project, which
-contains a CPython-based cppyy.py module (with similar dependencies as the
-one that comes with PyPy).
-A standalone pip-installable version is planned, but for now you can install
-ROOT through your favorite distribution installer (available in the science
-section).
-
-.. _PyROOT: https://root.cern.ch/pyroot
-
-There are a couple of minor differences between the two versions of cppyy
-(the CPython version has a few more features).
-Work is on-going to integrate the nightly tests of both to make sure their
-feature sets are equalized.
-
-
-Python3
--------
-
-The CPython version of cppyy supports Python3, assuming your packager has
-build the backend for it.
-The cppyy module has not been tested with the `Py3k`_ version of PyPy.
-Note that the generated reflection information (from ``genreflex``) is fully
-independent of Python, and does not need to be rebuild when switching versions
-or interpreters.
-
-.. _Py3k: https://bitbucket.org/pypy/pypy/src/py3k
-
-
-.. toctree::
-   :hidden:
-
-   cppyy_example
diff --git a/pypy/doc/cppyy_example.rst b/pypy/doc/cppyy_example.rst
deleted file mode 100644
--- a/pypy/doc/cppyy_example.rst
+++ /dev/null
@@ -1,59 +0,0 @@
-File example.h
-==============
-
-::
-
-    #include <iostream>
-    #include <vector>
-
-    class AbstractClass {
-    public:
-        virtual ~AbstractClass() {}
-        virtual void abstract_method() = 0;
-    };
-
-    class ConcreteClass : AbstractClass {
-    public:
-        ConcreteClass(int n=42) : m_int(n) {}
-        ~ConcreteClass() {}
-
-        virtual void abstract_method() {
-            std::cout << "called concrete method" << std::endl;
-        }
-
-        void array_method(int* ad, int size) {
-            for (int i=0; i < size; ++i)
-                std::cout << ad[i] << ' ';
-            std::cout << std::endl;
-        }
-
-        void array_method(double* ad, int size) {
-            for (int i=0; i < size; ++i)
-                std::cout << ad[i] << ' ';
-            std::cout << std::endl;
-        }
-
-        AbstractClass* show_autocast() {
-            return this;
-        }
-
-        operator const char*() {
-            return "Hello operator const char*!";
-        }
-
-    public:
-        int m_int;
-    };
-
-    namespace Namespace {
-
-       class ConcreteClass {
-       public:
-          class NestedClass {
-          public:
-             std::vector<int> m_v;
-          };
-
-       };
-
-    } // namespace Namespace
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -330,6 +330,8 @@
 
  - ``frozenset`` (empty frozenset only)
 
+ - unbound method objects (for Python 2 only)
+
 This change requires some changes to ``id`` as well. ``id`` fulfills the
 following condition: ``x is y <=> id(x) == id(y)``. Therefore ``id`` of the
 above types will return a value that is computed from the argument, and can
@@ -427,7 +429,8 @@
 
 * the ``__builtins__`` name is always referencing the ``__builtin__`` module,
   never a dictionary as it sometimes is in CPython. Assigning to
-  ``__builtins__`` has no effect.
+  ``__builtins__`` has no effect.  (For usages of tools like
+  RestrictedPython, see `issue #2653`_.)
 
 * directly calling the internal magic methods of a few built-in types
   with invalid arguments may have a slightly different result.  For
@@ -533,7 +536,12 @@
   or ``float`` subtypes. Currently PyPy does not support the
   ``__class__`` attribute assignment for any non heaptype subtype.
 
+* In PyPy, module and class dictionaries are optimized under the assumption
+  that deleting attributes from them are rare. Because of this, e.g.
+  ``del foo.bar`` where ``foo`` is a module (or class) that contains the
+  function ``bar``, is significantly slower than CPython.
+
 .. _`is ignored in PyPy`: http://bugs.python.org/issue14621
 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html
 .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/
-
+.. _`issue #2653`: https://bitbucket.org/pypy/pypy/issues/2653/
diff --git a/pypy/doc/extending.rst b/pypy/doc/extending.rst
--- a/pypy/doc/extending.rst
+++ b/pypy/doc/extending.rst
@@ -12,7 +12,7 @@
 
 * Write them in pure Python and use ctypes_.
 
-* Write them in C++ and bind them through  :doc:`cppyy <cppyy>` using Cling.
+* Write them in C++ and bind them through  cppyy_ using Cling.
 
 * Write them as `RPython mixed modules`_.
 
@@ -61,29 +61,22 @@
 .. _libffi: http://sourceware.org/libffi/
 
 
-Cling and cppyy
----------------
+cppyy
+-----
 
-The builtin :doc:`cppyy <cppyy>` module uses reflection information, provided by
-`Cling`_ (which needs to be `installed separately`_), of C/C++ code to
-automatically generate bindings at runtime.
-In Python, classes and functions are always runtime structures, so when they
-are generated matters not for performance.
-However, if the backend itself is capable of dynamic behavior, it is a much
-better functional match, allowing tighter integration and more natural
-language mappings.
+For C++, _cppyy_ is an automated bindings generator available for both
+PyPy and CPython.

From pypy.commits at gmail.com  Fri Sep 29 13:50:54 2017
From: pypy.commits at gmail.com (mjacob)
Date: Fri, 29 Sep 2017 10:50:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.6: hg merge py3.5
Message-ID: <59ce87fe.8e861c0a.198eb.b9f2@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.6
Changeset: r92514:d4a38e82d0d7
Date: 2017-09-29 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/d4a38e82d0d7/

Log:	hg merge py3.5

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -669,7 +669,7 @@
         'PySlice_Type': 'space.gettypeobject(W_SliceObject.typedef)',
         'PyStaticMethod_Type': 'space.gettypeobject(StaticMethod.typedef)',
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
-        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
+        'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)',
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)
@@ -1338,17 +1338,20 @@
     for decl in FORWARD_DECLS:
         decls[pypy_decl].append("%s;" % (decl,))
     decls[pypy_decl].append("""
-        /* hack for https://bugs.python.org/issue29943 */
-        PyAPI_FUNC(int) %s(PySliceObject *arg0,
-                           Signed arg1, Signed *arg2,
-                           Signed *arg3, Signed *arg4, Signed *arg5);
-        static int PySlice_GetIndicesEx(PySliceObject *arg0, Py_ssize_t arg1,
-                Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
-                Py_ssize_t *arg5) {
-            return %s(arg0, arg1, arg2, arg3,
-                      arg4, arg5);
-        }
-    """ % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
+/* hack for https://bugs.python.org/issue29943 */
+
+PyAPI_FUNC(int) %s(PyObject *arg0,
+                    Signed arg1, Signed *arg2,
+                    Signed *arg3, Signed *arg4, Signed *arg5);
+#ifdef __GNUC__
+__attribute__((__unused__))
+#endif
+static int PySlice_GetIndicesEx(PyObject *arg0, Py_ssize_t arg1,
+        Py_ssize_t *arg2, Py_ssize_t *arg3, Py_ssize_t *arg4,
+        Py_ssize_t *arg5) {
+    return %s(arg0, arg1, arg2, arg3,
+                arg4, arg5);
+}""" % ((mangle_name(prefix, 'PySlice_GetIndicesEx'),)*2))
 
     for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems():
         header = decls[header_name]
diff --git a/pypy/module/cpyext/include/descrobject.h b/pypy/module/cpyext/include/descrobject.h
--- a/pypy/module/cpyext/include/descrobject.h
+++ b/pypy/module/cpyext/include/descrobject.h
@@ -1,34 +1,6 @@
 #ifndef Py_DESCROBJECT_H
 #define Py_DESCROBJECT_H
 
-#define PyDescr_COMMON \
-    PyObject_HEAD \
-    PyTypeObject *d_type; \
-    PyObject *d_name
-
-typedef struct {
-    PyDescr_COMMON;
-} PyDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyMethodDef *d_method;
-} PyMethodDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct PyMemberDef *d_member;
-} PyMemberDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    PyGetSetDef *d_getset;
-} PyGetSetDescrObject;
-
-typedef struct {
-    PyDescr_COMMON;
-    struct wrapperbase *d_base;
-    void *d_wrapped; /* This can be any function pointer */
-} PyWrapperDescrObject;
+#include "cpyext_descrobject.h"
 
 #endif
diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py
--- a/pypy/module/cpyext/methodobject.py
+++ b/pypy/module/cpyext/methodobject.py
@@ -1,4 +1,4 @@
-from rpython.rtyper.lltypesystem import lltype, rffi, llmemory
+from rpython.rtyper.lltypesystem import lltype, rffi
 
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError, oefmt
@@ -10,8 +10,8 @@
 from pypy.module.cpyext.api import (
     CONST_STRING, METH_CLASS, METH_COEXIST, METH_KEYWORDS, METH_NOARGS, METH_O,
     METH_STATIC, METH_VARARGS, PyObject, bootstrap_function,
-    cpython_api, generic_cpy_call, CANNOT_FAIL,
-    PyTypeObjectPtr, slot_function, cts)
+    cpython_api, generic_cpy_call, CANNOT_FAIL, slot_function, cts,
+    build_type_checkers)
 from pypy.module.cpyext.pyobject import (
     Py_DecRef, from_ref, make_ref, as_pyobj, make_typedescr)
 
@@ -102,7 +102,7 @@
         return self.space.unwrap(self.descr_method_repr())
 
     def descr_method_repr(self):
-        w_objclass = self.w_objclass 
+        w_objclass = self.w_objclass
         assert isinstance(w_objclass, W_TypeObject)
         return self.space.newtext("<method '%s' of '%s' objects>" % (
             self.name, w_objclass.name))
@@ -110,7 +110,7 @@
     def descr_call(self, space, __args__):
         args_w, kw_w = __args__.unpack()
         if len(args_w) < 1:
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%8' of '%s' object needs an argument",
@@ -118,7 +118,7 @@
         w_instance = args_w[0]
         # XXX: needs a stricter test
         if not space.isinstance_w(w_instance, self.w_objclass):
-            w_objclass = self.w_objclass 
+            w_objclass = self.w_objclass
             assert isinstance(w_objclass, W_TypeObject)
             raise oefmt(space.w_TypeError,
                 "descriptor '%8' requires a '%s' object but received a '%T'",
@@ -130,6 +130,10 @@
         ret = self.call(space, w_instance, w_args, w_kw)
         return ret
 
+# PyPy addition, for Cython
+_, _ = build_type_checkers("MethodDescr", W_PyCMethodObject)
+
+
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyCFunction_Check(space, w_obj):
     from pypy.interpreter.function import BuiltinFunction
@@ -156,6 +160,7 @@
             (self.name.decode('utf-8'), self.w_objclass.getname(self.space)))
 
 
+
 class W_PyCWrapperObject(W_Root):
     def __init__(self, space, pto, method_name, wrapper_func,
                  wrapper_func_kwds, doc, func, offset=None):
@@ -323,11 +328,15 @@
 def PyClassMethod_New(space, w_func):
     return ClassMethod(w_func)
 
- at cpython_api([PyTypeObjectPtr, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewMethod(space, w_type, method):
     return W_PyCMethodObject(space, method, w_type)
 
- at cpython_api([PyObject, lltype.Ptr(PyMethodDef)], PyObject)
+ at cts.decl("""
+    PyObject *
+    PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method)""")
 def PyDescr_NewClassMethod(space, w_type, method):
     return W_PyCClassMethodObject(space, method, w_type)
 
diff --git a/pypy/module/cpyext/parse/cpyext_descrobject.h b/pypy/module/cpyext/parse/cpyext_descrobject.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/parse/cpyext_descrobject.h
@@ -0,0 +1,29 @@
+typedef struct {
+    PyObject_HEAD
+    PyTypeObject *d_type;
+    PyObject *d_name;
+    PyObject *d_qualname;
+} PyDescrObject;
+
+#define PyDescr_COMMON PyDescrObject d_common
+
+typedef struct {
+    PyDescr_COMMON;
+    PyMethodDef *d_method;
+} PyMethodDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct PyMemberDef *d_member;
+} PyMemberDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    PyGetSetDef *d_getset;
+} PyGetSetDescrObject;
+
+typedef struct {
+    PyDescr_COMMON;
+    struct wrapperbase *d_base;
+    void *d_wrapped; /* This can be any function pointer */
+} PyWrapperDescrObject;
diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c
--- a/pypy/module/cpyext/test/foo.c
+++ b/pypy/module/cpyext/test/foo.c
@@ -83,6 +83,35 @@
     return cls;
 }
 
+// for CPython
+#ifndef PyMethodDescr_Check
+int PyMethodDescr_Check(PyObject* method)
+{
+    PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append");
+    if (!meth) return 0;
+    int res = PyObject_TypeCheck(method, meth->ob_type);
+    Py_DECREF(meth);
+    return res;
+}
+#endif
+
+PyObject* make_classmethod(PyObject* method)
+{
+    // adapted from __Pyx_Method_ClassMethod
+    if (PyMethodDescr_Check(method)) {
+        PyMethodDescrObject *descr = (PyMethodDescrObject *)method;
+        PyTypeObject *d_type = descr->d_common.d_type;
+        return PyDescr_NewClassMethod(d_type, descr->d_method);
+    }
+    else if (PyMethod_Check(method)) {
+        return PyClassMethod_New(PyMethod_GET_FUNCTION(method));
+    }
+    else {
+        PyErr_SetString(PyExc_TypeError, "unknown method kind");
+        return NULL;
+    }
+}
+
 static PyObject *
 foo_unset(fooobject *self)
 {
@@ -95,6 +124,7 @@
     {"copy",      (PyCFunction)foo_copy,      METH_NOARGS,  NULL},
     {"create",    (PyCFunction)foo_create,    METH_NOARGS|METH_STATIC,  NULL},
     {"classmeth", (PyCFunction)foo_classmeth, METH_NOARGS|METH_CLASS,  NULL},
+    {"fake_classmeth", (PyCFunction)foo_classmeth, METH_NOARGS,  NULL},
     {"unset_string_member", (PyCFunction)foo_unset, METH_NOARGS, NULL},
     {NULL, NULL}                 /* sentinel */
 };
@@ -167,19 +197,19 @@
     /* copied from numpy scalartypes.c for inherited classes */
     if (t->tp_bases && (PyTuple_GET_SIZE(t->tp_bases) > 1))
     {
-        PyTypeObject *sup; 
-        /* We are inheriting from a Python type as well so 
+        PyTypeObject *sup;
+        /* We are inheriting from a Python type as well so
            give it first dibs on conversion */
         sup = (PyTypeObject *)PyTuple_GET_ITEM(t->tp_bases, 1);
-        /* Prevent recursion */ 
-        if (new_fooType != sup->tp_new) 
+        /* Prevent recursion */
+        if (new_fooType != sup->tp_new)
         {
             o = sup->tp_new(t, args, kwds);
             return o;
         }
     }
     o = t->tp_alloc(t, 0);
-    return o;   
+    return o;
 };
 
 static PyMemberDef foo_members[] = {
@@ -717,7 +747,7 @@
     "foo",
     "Module Doc",
     -1,
-    foo_functions, 
+    foo_functions,
     NULL,
     NULL,
     NULL,
@@ -751,6 +781,7 @@
 #endif
 {
     PyObject *d;
+    PyObject *fake_classmeth, *classmeth;
 #if PY_MAJOR_VERSION >= 3
     PyObject *module = PyModule_Create(&moduledef);
 #else
@@ -808,6 +839,12 @@
         INITERROR;
     gettype2 = PyObject_New(PyObject, &GetType2);
 
+    fake_classmeth = PyDict_GetItemString((PyObject *)fooType.tp_dict, "fake_classmeth");
+    classmeth = make_classmethod(fake_classmeth);
+    if (classmeth == NULL)
+        INITERROR;
+    if (PyDict_SetItemString((PyObject *)fooType.tp_dict, "fake_classmeth", classmeth) < 0)
+        INITERROR;
 
     d = PyModule_GetDict(module);
     if (d == NULL)
diff --git a/pypy/module/cpyext/test/test_fileobject.py b/pypy/module/cpyext/test/test_fileobject.py
--- a/pypy/module/cpyext/test/test_fileobject.py
+++ b/pypy/module/cpyext/test/test_fileobject.py
@@ -7,7 +7,7 @@
         module = self.import_extension('foo', [
             ("defenc", "METH_NOARGS",
              """
-                return PyString_FromString(Py_FileSystemDefaultEncoding);
+                return PyUnicode_FromString(Py_FileSystemDefaultEncoding);
              """),
             ])
         assert module.defenc() == sys.getfilesystemencoding()
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -135,6 +135,12 @@
             "<method 'copy' of 'foo' objects>")
         raises(TypeError, descr, None)
 
+    def test_cython_fake_classmethod(self):
+        module = self.import_module(name='foo')
+        print(module.fooType.fake_classmeth)
+        print(type(module.fooType.fake_classmeth))
+        assert module.fooType.fake_classmeth() is module.fooType
+
     def test_new(self):
         # XXX cpython segfaults but if run singly (with -k test_new) this passes
         module = self.import_module(name='foo')
@@ -1315,6 +1321,47 @@
         assert Asize == Bsize
         assert Asize > basesize
 
+    def test_multiple_inheritance_bug1(self):
+        module = self.import_extension('foo', [
+           ("get_type", "METH_NOARGS",
+            '''
+                Py_INCREF(&Foo_Type);
+                return (PyObject *)&Foo_Type;
+            '''
+            ), ("forty_two", "METH_O",
+            '''
+                return PyLong_FromLong(42);
+            '''
+            )], prologue='''
+            static PyTypeObject Foo_Type = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "foo.foo",
+            };
+            static PyObject *dummy_new(PyTypeObject *t, PyObject *a,
+                                       PyObject *k)
+            {
+                abort();   /* never actually called in CPython */
+            }
+            ''', more_init = '''
+                Foo_Type.tp_base = (PyTypeObject *)PyExc_Exception;
+                Foo_Type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+                Foo_Type.tp_new = dummy_new;
+                if (PyType_Ready(&Foo_Type) < 0) INITERROR;
+            ''')
+        Foo = module.get_type()
+        class A(Foo, SyntaxError):
+            pass
+        assert A.__base__ is SyntaxError
+        A(42)    # assert is not aborting
+
+        class Bar(Exception):
+            __new__ = module.forty_two
+
+        class B(Bar, SyntaxError):
+            pass
+
+        assert B() == 42
+
 
 class AppTestHashable(AppTestCpythonExtensionBase):
     def test_unhashable(self):
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -12,7 +12,7 @@
 from pypy.module.cpyext.api import (
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t,
     slot_function, generic_cpy_call, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers_flags, cts, parse_dir, PyObjectFields, PyTypeObject,
+    build_type_checkers_flags, cts, parse_dir, PyTypeObject,
     PyTypeObjectPtr, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, Py_TPFLAGS_READY, Py_TPFLAGS_READYING,
     Py_TPFLAGS_LONG_SUBCLASS, Py_TPFLAGS_LIST_SUBCLASS,
@@ -39,12 +39,14 @@
 from pypy.objspace.std.typeobject import W_TypeObject, find_best_base
 
 
+
 #WARN_ABOUT_MISSING_SLOT_FUNCTIONS = False
 
 PyType_Check, PyType_CheckExact = build_type_checkers_flags("Type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
 
+cts.parse_header(parse_dir / "cpyext_descrobject.h")
 cts.parse_header(parse_dir / "typeslots.h")
 
 
@@ -115,57 +117,24 @@
     )
 assert not W_MemberDescr.typedef.acceptable_as_base_class  # no __new__
 
-PyDescrObject = lltype.ForwardReference()
-PyDescrObjectPtr = lltype.Ptr(PyDescrObject)
-PyDescrObjectFields = PyObjectFields + (
-    ("d_type", PyTypeObjectPtr),
-    ("d_name", PyObject),
-    )
-cpython_struct("PyDescrObject", PyDescrObjectFields,
-               PyDescrObject)
-
-PyMemberDescrObjectStruct = lltype.ForwardReference()
-PyMemberDescrObject = lltype.Ptr(PyMemberDescrObjectStruct)
-PyMemberDescrObjectFields = PyDescrObjectFields + (
-    ("d_member", lltype.Ptr(PyMemberDef)),
-    )
-cpython_struct("PyMemberDescrObject", PyMemberDescrObjectFields,
-               PyMemberDescrObjectStruct, level=2)
-
-PyGetSetDescrObjectStruct = lltype.ForwardReference()
-PyGetSetDescrObject = lltype.Ptr(PyGetSetDescrObjectStruct)
-PyGetSetDescrObjectFields = PyDescrObjectFields + (
-    ("d_getset", lltype.Ptr(PyGetSetDef)),
-    )
-cpython_struct("PyGetSetDescrObject", PyGetSetDescrObjectFields,
-               PyGetSetDescrObjectStruct, level=2)
-
-PyMethodDescrObjectStruct = lltype.ForwardReference()
-PyMethodDescrObject = lltype.Ptr(PyMethodDescrObjectStruct)
-PyMethodDescrObjectFields = PyDescrObjectFields + (
-    ("d_method", lltype.Ptr(PyMethodDef)),
-    )
-cpython_struct("PyMethodDescrObject", PyMethodDescrObjectFields,
-               PyMethodDescrObjectStruct, level=2)
-
 @bootstrap_function
 def init_memberdescrobject(space):
     make_typedescr(W_MemberDescr.typedef,
-                   basestruct=PyMemberDescrObject.TO,
+                   basestruct=cts.gettype('PyMemberDescrObject'),
                    attach=memberdescr_attach,
                    realize=memberdescr_realize,
                    )
     make_typedescr(W_GetSetPropertyEx.typedef,
-                   basestruct=PyGetSetDescrObject.TO,
+                   basestruct=cts.gettype('PyGetSetDescrObject'),
                    attach=getsetdescr_attach,
                    )
     make_typedescr(W_PyCClassMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=classmethoddescr_realize,
                    )
     make_typedescr(W_PyCMethodObject.typedef,
-                   basestruct=PyMethodDescrObject.TO,
+                   basestruct=cts.gettype('PyMethodDescrObject'),
                    attach=methoddescr_attach,
                    realize=methoddescr_realize,
                    )
@@ -175,7 +144,7 @@
     Fills a newly allocated PyMemberDescrObject with the given W_MemberDescr
     object. The values must not be modified.
     """
-    py_memberdescr = rffi.cast(PyMemberDescrObject, py_obj)
+    py_memberdescr = cts.cast('PyMemberDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_MemberDescr)
     py_memberdescr.c_d_member = w_obj.member
@@ -194,7 +163,7 @@
     Fills a newly allocated PyGetSetDescrObject with the given W_GetSetPropertyEx
     object. The values must not be modified.
     """
-    py_getsetdescr = rffi.cast(PyGetSetDescrObject, py_obj)
+    py_getsetdescr = cts.cast('PyGetSetDescrObject*', py_obj)
     if isinstance(w_obj, GetSetProperty):
         py_getsetdef = make_GetSet(space, w_obj)
         assert space.isinstance_w(w_userdata, space.w_type)
@@ -206,7 +175,7 @@
     py_getsetdescr.c_d_getset = w_obj.getset
 
 def methoddescr_attach(space, py_obj, w_obj, w_userdata=None):
-    py_methoddescr = rffi.cast(PyMethodDescrObject, py_obj)
+    py_methoddescr = cts.cast('PyMethodDescrObject*', py_obj)
     # XXX assign to d_dname, d_type?
     assert isinstance(w_obj, W_PyCFunctionObject)
     py_methoddescr.c_d_method = w_obj.ml
@@ -426,6 +395,9 @@
     ptr = get_new_method_def(space)
     ptr.c_ml_meth = rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 
+def is_tp_new_wrapper(space, ml):
+    return ml.c_ml_meth == rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
+
 def add_tp_new_wrapper(space, dict_w, pto):
     if "__new__" in dict_w:
         return
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -641,6 +641,12 @@
             if w_newdescr is None:    # see test_crash_mro_without_object_1
                 raise oefmt(space.w_TypeError, "cannot create '%N' instances",
                             self)
+            #
+            # issue #2666
+            if space.config.objspace.usemodules.cpyext:
+                w_newtype, w_newdescr = self.hack_which_new_to_call(
+                    w_newtype, w_newdescr)
+            #
             w_newfunc = space.get(w_newdescr, space.w_None, w_type=self)
             if (space.config.objspace.std.newshortcut and
                 not we_are_jitted() and
@@ -661,6 +667,30 @@
                                 "__init__() should return None")
         return w_newobject
 
+    def hack_which_new_to_call(self, w_newtype, w_newdescr):
+        # issue #2666: for cpyext, we need to hack in order to reproduce
+        # an "optimization" of CPython that actually changes behaviour
+        # in corner cases.
+        #
+        # * Normally, we use the __new__ found in the MRO in the normal way.
+        #
+        # * If by chance this __new__ happens to be implemented as a C
+        #   function, then instead, we discard it and use directly
+        #   self.__base__.tp_new.
+        #
+        # * Most of the time this is the same (and faster for CPython), but
+        #   it can fail if self.__base__ happens not to be the first base.
+        #
+        from pypy.module.cpyext.methodobject import W_PyCFunctionObject
+        from pypy.module.cpyext.typeobject import is_tp_new_wrapper
+
+        if (isinstance(w_newdescr, W_PyCFunctionObject) and
+                is_tp_new_wrapper(self.space, w_newdescr.ml)):
+            w_bestbase = find_best_base(self.bases_w)
+            return w_bestbase.lookup_where('__new__')
+        else:
+            return w_newtype, w_newdescr
+
     def descr_repr(self, space):
         w_mod = self.get_module()
         if w_mod is None or not space.isinstance_w(w_mod, space.w_text):
diff --git a/pypy/pytest-A.cfg b/pypy/pytest-A.cfg
--- a/pypy/pytest-A.cfg
+++ b/pypy/pytest-A.cfg
@@ -1,5 +1,8 @@
+import sys
 cherrypick = ['interpreter', 'objspace/test', 'objspace/std', 'module']
 
 interp = ['python']
-test_driver = ['test_all.py', '-A', '--python=goal/pypy3-c']
-
+if sys.platform == 'win32':
+    test_driver = ['test_all.py', '-A', '--python=goal/pypy3-cw.exe']
+else:
+    test_driver = ['test_all.py', '-A', '--python=goal/pypy3-c']

From pypy.commits at gmail.com  Fri Sep 29 13:52:03 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 29 Sep 2017 10:52:03 -0700 (PDT)
Subject: [pypy-commit] pypy memory-accounting: change back to 10M seems less
 insane
Message-ID: <59ce8843.44231c0a.f8f94.7b84@mx.google.com>

Author: fijal
Branch: memory-accounting
Changeset: r92515:ddd3bbb41771
Date: 2017-09-29 19:50 +0200
http://bitbucket.org/pypy/pypy/changeset/ddd3bbb41771/

Log:	change back to 10M seems less insane

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1317,7 +1317,7 @@
 
         self = space.allocate_instance(_SSLContext, w_subtype)
         assert isinstance(self, _SSLContext)
-        rgc.add_memory_pressure(10 * 1024, self)
+        rgc.add_memory_pressure(10 * 1024 * 1024, self)
         self.ctx = ctx
         self.check_hostname = False
         self.register_finalizer(space)

From pypy.commits at gmail.com  Fri Sep 29 13:53:02 2017
From: pypy.commits at gmail.com (fijal)
Date: Fri, 29 Sep 2017 10:53:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: bump the limit back
Message-ID: <59ce887e.8298df0a.a298.107c@mx.google.com>

Author: fijal
Branch: 
Changeset: r92516:1721ac9f04a9
Date: 2017-09-29 19:52 +0200
http://bitbucket.org/pypy/pypy/changeset/1721ac9f04a9/

Log:	bump the limit back

diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py
--- a/pypy/module/_ssl/interp_ssl.py
+++ b/pypy/module/_ssl/interp_ssl.py
@@ -1316,7 +1316,7 @@
         if not ctx:
             raise ssl_error(space, "failed to allocate SSL context")
 
-        rgc.add_memory_pressure(10 * 1024)
+        rgc.add_memory_pressure(10 * 1024 * 1024)
         self = space.allocate_instance(_SSLContext, w_subtype)
         self.ctx = ctx
         self.check_hostname = False

From pypy.commits at gmail.com  Fri Sep 29 14:44:23 2017
From: pypy.commits at gmail.com (mjacob)
Date: Fri, 29 Sep 2017 11:44:23 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix vmprof test.
Message-ID: <59ce9487.d0e61c0a.d92a3.52c4@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r92517:42233a3efdd4
Date: 2017-09-29 20:44 +0200
http://bitbucket.org/pypy/pypy/changeset/42233a3efdd4/

Log:	Fix vmprof test.

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -45,10 +45,10 @@
                     _, size = struct.unpack("ll", s[i:i + 2 * WORD])
                     count += 1
                     i += 2 * WORD + size
-                elif s[i] == '\x06':
+                elif s[i] == 6:
                     print(s[i:i+24])
                     i += 1+8+8+8
-                elif s[i] == '\x07':
+                elif s[i] == 7:
                     i += 1
                     # skip string
                     size, = struct.unpack("l", s[i:i + WORD])

From pypy.commits at gmail.com  Fri Sep 29 16:36:57 2017
From: pypy.commits at gmail.com (pjenvey)
Date: Fri, 29 Sep 2017 13:36:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: update per 1721ac9f04a9
Message-ID: <59ceaee9.5ce81c0a.ebd62.c291@mx.google.com>

Author: Philip Jenvey <pjenvey at underboss.org>
Branch: 
Changeset: r92518:f5e9d7a6acf1
Date: 2017-09-29 13:36 -0700
http://bitbucket.org/pypy/pypy/changeset/f5e9d7a6acf1/

Log:	update per 1721ac9f04a9

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -148,8 +148,7 @@
   * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
   * Replace magical NOT RPYTHON comment with a decorator
   * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
-  * Reduce excessive ``memory_pressure`` for ``_SSLContext`` objects and add
-    ``memory_pressure`` for ``_SSLSocket`` objects
+  * Add ``memory_pressure`` for ``_SSLSocket`` objects
 
 * Degredations
 

From pypy.commits at gmail.com  Fri Sep 29 17:40:36 2017
From: pypy.commits at gmail.com (antocuni)
Date: Fri, 29 Sep 2017 14:40:36 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-jit: don't look inside this for now, else
 it raises InvalidCast;
 probably it is possible to work-around it by casting both operands to void*,
 but didn't try yet
Message-ID: <59cebdd4.02addf0a.50959.dd78@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: cpyext-jit
Changeset: r92519:31cea4ce3c8c
Date: 2017-09-29 21:41 +0100
http://bitbucket.org/pypy/pypy/changeset/31cea4ce3c8c/

Log:	don't look inside this for now, else it raises InvalidCast; probably
	it is possible to work-around it by casting both operands to void*,
	but didn't try yet

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -394,6 +394,7 @@
     ptr = get_new_method_def(space)
     ptr.c_ml_meth = rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 
+ at jit.dont_look_inside
 def is_tp_new_wrapper(space, ml):
     return ml.c_ml_meth == rffi.cast(PyCFunction, llslot(space, tp_new_wrapper))
 

From pypy.commits at gmail.com  Fri Sep 29 18:40:30 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 15:40:30 -0700 (PDT)
Subject: [pypy-commit] pypy default: Expose the Descr_Types,
 like on CPython 3.*
Message-ID: <59cecbde.88b0df0a.948c5.b4cd@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r92520:fb1ae9b91f2f
Date: 2017-09-30 00:39 +0200
http://bitbucket.org/pypy/pypy/changeset/fb1ae9b91f2f/

Log:	Expose the Descr_Types, like on CPython 3.*

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -654,6 +654,10 @@
         'PyClass_Type': 'space.gettypeobject(W_ClassObject.typedef)',
         'PyStaticMethod_Type': 'space.gettypeobject(StaticMethod.typedef)',
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
+        'PyClassMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCClassMethodObject.typedef)',
+        'PyGetSetDescr_Type': 'space.gettypeobject(cpyext.typeobject.W_GetSetPropertyEx.typedef)',
+        'PyMemberDescr_Type': 'space.gettypeobject(cpyext.typeobject.W_MemberDescr.typedef)',
+        'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
         'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         }.items():
         register_global(cpyname, 'PyTypeObject*', pypyexpr, header=pypy_decl)

From pypy.commits at gmail.com  Fri Sep 29 18:42:40 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 29 Sep 2017 15:42:40 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <59cecc60.c9331c0a.1d716.caa4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r92521:45380b7b402f
Date: 2017-09-30 00:42 +0200
http://bitbucket.org/pypy/pypy/changeset/45380b7b402f/

Log:	hg merge default

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -148,8 +148,7 @@
   * Issue 2590_: fix the bounds in the GC when allocating a lot of objects with finalizers
   * Replace magical NOT RPYTHON comment with a decorator
   * Implement ``socket.sendmsg()``/``.recvmsg()`` for py3.5
-  * Reduce excessive ``memory_pressure`` for ``_SSLContext`` objects and add
-    ``memory_pressure`` for ``_SSLSocket`` objects
+  * Add ``memory_pressure`` for ``_SSLSocket`` objects
 
 * Degredations
 
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -669,6 +669,10 @@
         'PySlice_Type': 'space.gettypeobject(W_SliceObject.typedef)',
         'PyStaticMethod_Type': 'space.gettypeobject(StaticMethod.typedef)',
         'PyCFunction_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCFunctionObject.typedef)',
+        'PyClassMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCClassMethodObject.typedef)',
+        'PyGetSetDescr_Type': 'space.gettypeobject(cpyext.typeobject.W_GetSetPropertyEx.typedef)',
+        'PyMemberDescr_Type': 'space.gettypeobject(cpyext.typeobject.W_MemberDescr.typedef)',
+        'PyMethodDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCMethodObject.typedef)',
         'PyWrapperDescr_Type': 'space.gettypeobject(cpyext.methodobject.W_PyCWrapperObject.typedef)',
         'PyInstanceMethod_Type': 'space.gettypeobject(cpyext.classobject.InstanceMethod.typedef)',
         }.items():
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -1362,6 +1362,11 @@
 
         assert B() == 42
 
+        # aaaaa even more hackiness
+        class C(A):
+            pass
+        C(42)   # assert is not aborting
+
 
 class AppTestHashable(AppTestCpythonExtensionBase):
     def test_unhashable(self):
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -682,14 +682,30 @@
         #   it can fail if self.__base__ happens not to be the first base.
         #
         from pypy.module.cpyext.methodobject import W_PyCFunctionObject
+
+        if isinstance(w_newdescr, W_PyCFunctionObject):
+            return self._really_hack_which_new_to_call(w_newtype, w_newdescr)
+        else:
+            return w_newtype, w_newdescr
+
+    def _really_hack_which_new_to_call(self, w_newtype, w_newdescr):
+        # This logic is moved in yet another helper function that
+        # is recursive.  We call this only if we see a
+        # W_PyCFunctionObject.  That's a performance optimization
+        # because in the common case, we won't call any function that
+        # contains the stack checks.
+        from pypy.module.cpyext.methodobject import W_PyCFunctionObject
         from pypy.module.cpyext.typeobject import is_tp_new_wrapper
 
         if (isinstance(w_newdescr, W_PyCFunctionObject) and
+                w_newtype is not self and
                 is_tp_new_wrapper(self.space, w_newdescr.ml)):
             w_bestbase = find_best_base(self.bases_w)
-            return w_bestbase.lookup_where('__new__')
-        else:
-            return w_newtype, w_newdescr
+            if w_bestbase is not None:
+                w_newtype, w_newdescr = w_bestbase.lookup_where('__new__')
+                return w_bestbase._really_hack_which_new_to_call(w_newtype,
+                                                                 w_newdescr)
+        return w_newtype, w_newdescr
 
     def descr_repr(self, space):
         w_mod = self.get_module()

From pypy.commits at gmail.com  Sat Sep 30 02:19:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 23:19:01 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #337
Message-ID: <59cf3755.84881c0a.82bd3.c71b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3033:9d9bee10c792
Date: 2017-09-30 08:18 +0200
http://bitbucket.org/cffi/cffi/changeset/9d9bee10c792/

Log:	Issue #337

	Fixes for test failures and warnings on ARM

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2110,7 +2110,10 @@
     if sys.platform.startswith("linux"):
         BWChar = new_primitive_type("wchar_t")
         assert sizeof(BWChar) == 4
-        assert int(cast(BWChar, -1)) == -1        # signed, on linux
+        if platform.machine().startswith(('arm', 'aarch64')):
+            assert int(cast(BWChar, -1)) == 4294967295 # unsigned, on ARM
+        else:
+            assert int(cast(BWChar, -1)) == -1     # "often" signed...
 
 def test_char16():
     BChar16 = new_primitive_type("char16_t")
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -238,9 +238,9 @@
 _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
 {
     if (sizeof(_cffi_wchar_t) == 2)
-        return _cffi_from_c_wchar_t(x);
+        return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
-        return _cffi_from_c_wchar3216_t(x);
+        return _cffi_from_c_wchar3216_t((int)x);
 }
 
 _CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
@@ -254,7 +254,7 @@
 _CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
 {
     if (sizeof(_cffi_wchar_t) == 4)
-        return _cffi_from_c_wchar_t(x);
+        return _cffi_from_c_wchar_t((_cffi_wchar_t)x);
     else
         return _cffi_from_c_wchar3216_t(x);
 }

From pypy.commits at gmail.com  Sat Sep 30 02:34:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 29 Sep 2017 23:34:51 -0700 (PDT)
Subject: [pypy-commit] cffi default: ImportError
Message-ID: <59cf3b0b.4eae1c0a.309a7.973a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r3034:7a5717e07bca
Date: 2017-09-30 08:34 +0200
http://bitbucket.org/cffi/cffi/changeset/7a5717e07bca/

Log:	ImportError

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2110,6 +2110,7 @@
     if sys.platform.startswith("linux"):
         BWChar = new_primitive_type("wchar_t")
         assert sizeof(BWChar) == 4
+        import platform
         if platform.machine().startswith(('arm', 'aarch64')):
             assert int(cast(BWChar, -1)) == 4294967295 # unsigned, on ARM
         else:

From pypy.commits at gmail.com  Sat Sep 30 17:49:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 30 Sep 2017 14:49:45 -0700 (PDT)
Subject: [pypy-commit] pypy default: note that cython at least 0.27.1 is
 required
Message-ID: <59d01179.9c471c0a.3bfde.74cf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r92522:3450d3c2b444
Date: 2017-10-01 00:48 +0300
http://bitbucket.org/pypy/pypy/changeset/3450d3c2b444/

Log:	note that cython at least 0.27.1 is required

diff --git a/pypy/doc/release-v5.9.0.rst b/pypy/doc/release-v5.9.0.rst
--- a/pypy/doc/release-v5.9.0.rst
+++ b/pypy/doc/release-v5.9.0.rst
@@ -10,15 +10,18 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-NumPy and Pandas now work on PyPy2.7. Issues that appeared as excessive memory
+NumPy and Pandas now work on PyPy2.7 (together with Cython 0.27.1). Issues
+that appeared as excessive memory
 use were cleared up and other incompatibilities were resolved. The C-API
 compatibility layer does slow down code which crosses the python-c interface
 often, we have ideas on how it could be improved, and still recommend
 using pure python on PyPy or interfacing via CFFI_. Many other modules
 based on C-API exentions now work on PyPy as well.
 
-Cython 0.27 (released last week) should support more projects with PyPy, both
-on PyPy2.7 and PyPy3.5 beta.
+Cython 0.27.1 (released very recently) supports more projects with PyPy, both
+on PyPy2.7 and PyPy3.5 beta. Note version **0.27.1** is now the minimum
+version that supports this version of PyPy, due to some interactions with
+updated C-API interface code.
 
 We optimized the JSON parser for recurring string keys, which should decrease
 memory use to 50% and increase parsing speed by up to 15% for large JSON files