From pypy.commits at gmail.com  Thu Jun  1 04:34:30 2017
From: pypy.commits at gmail.com (danchr)
Date: Thu, 01 Jun 2017 01:34:30 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix translation on OS X (and likely
 FreeBSD as well)
Message-ID: <592fd196.d4a9df0a.3454e.ff0c@mx.google.com>

Author: Dan Villiom Podlaski Christiansen <dan at magenta.dk>
Branch: py3.5
Changeset: r91479:86914a929334
Date: 2017-05-31 22:05 +0200
http://bitbucket.org/pypy/pypy/changeset/86914a929334/

Log:	Fix translation on OS X (and likely FreeBSD as well)

	BSDs don't provide SCHED_BATCH, so remove the assert that checks for
	its presence.

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -235,8 +235,8 @@
         interpleveldefs['sched_get_priority_min'] = 'interp_posix.sched_get_priority_min'
         for _name in ['SCHED_FIFO', 'SCHED_RR', 'SCHED_OTHER',
         'SCHED_BATCH']:
-            assert getattr(rposix, _name) is not None, "missing %r" % (_name,)
-            interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
+            if getattr(rposix, _name) is not None:
+                interpleveldefs[_name] = 'space.wrap(%d)' % getattr(rposix, _name)
 
     for _name in ["O_CLOEXEC"]:
         if getattr(rposix, _name) is not None:
diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py
--- a/pypy/module/posix/test/test_posix2.py
+++ b/pypy/module/posix/test/test_posix2.py
@@ -944,7 +944,8 @@
             assert posix.sched_get_priority_max(posix.SCHED_FIFO) != -1
             assert posix.sched_get_priority_max(posix.SCHED_RR) != -1
             assert posix.sched_get_priority_max(posix.SCHED_OTHER) != -1
-            assert posix.sched_get_priority_max(posix.SCHED_BATCH) != -1
+            if getattr(posix, 'SCHED_BATCH', None):
+                assert posix.sched_get_priority_max(posix.SCHED_BATCH) != -1
 
     if hasattr(rposix, 'sched_get_priority_min'):
         def test_os_sched_get_priority_min(self):
@@ -953,7 +954,8 @@
             assert posix.sched_get_priority_min(posix.SCHED_FIFO) != -1
             assert posix.sched_get_priority_min(posix.SCHED_RR) != -1
             assert posix.sched_get_priority_min(posix.SCHED_OTHER) != -1
-            assert posix.sched_get_priority_min(posix.SCHED_BATCH) != -1
+            if getattr(posix, 'SCHED_BATCH', None):
+                assert posix.sched_get_priority_min(posix.SCHED_BATCH) != -1
             
     if hasattr(rposix, 'sched_get_priority_min'):
         def test_os_sched_priority_max_greater_than_min(self):

From pypy.commits at gmail.com  Thu Jun  1 08:53:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 01 Jun 2017 05:53:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix syntax error (oops!)
Message-ID: <59300e2f.b885df0a.cad2b.6cc5@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91480:002279b09536
Date: 2017-06-01 13:52 +0100
http://bitbucket.org/pypy/pypy/changeset/002279b09536/

Log:	fix syntax error (oops!)

diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -831,7 +831,7 @@
     stdlib_path = sys.pypy_find_stdlib(executable)
     if stdlib_path is None:
         initstdio()
-        print(STDLIB_WARNING % (getattr(sys, 'prefix', '<missing>'), 
+        print(STDLIB_WARNING % (getattr(sys, 'prefix', '<missing>'),),
             file=sys.stderr)
     else:
         sys.path[:] = stdlib_path
@@ -954,3 +954,4 @@
         import os; os.environ.update(reset)
         assert old_argv is sys.argv
         assert old_path is sys.path
+

From pypy.commits at gmail.com  Thu Jun  1 09:23:12 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Jun 2017 06:23:12 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak release notice,
 assume cffi 1.11 will be released in parallel
Message-ID: <59301540.c6be190a.253b1.86aa@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91481:4db703b9f72e
Date: 2017-06-01 16:22 +0300
http://bitbucket.org/pypy/pypy/changeset/4db703b9f72e/

Log:	tweak release notice, assume cffi 1.11 will be released in parallel

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -8,8 +8,7 @@
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
-PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
-3.5.3.
+PyPy3.5 includes the upstream stdlib version 3.5.3.
 
 We continue to make incremental improvements to our C-API
 compatibility layer (cpyext). PyPy2 can now import and run many C-extension
@@ -19,13 +18,13 @@
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
 release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
 as `these benchmarks show`_ it already gives a nice speed bump.
 We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
-CFFI_ has been updated to 1.10, improving an already great package for
+CFFI_ has been updated to 1.11, improving an already great package for
 interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the

From pypy.commits at gmail.com  Thu Jun  1 09:27:02 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 01 Jun 2017 06:27:02 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge default into release
Message-ID: <59301626.68d7190a.4c655.9aa7@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91482:c0b4e6c36ee6
Date: 2017-06-01 16:25 +0300
http://bitbucket.org/pypy/pypy/changeset/c0b4e6c36ee6/

Log:	merge default into release

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -734,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1187,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1479,6 +1486,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -8,8 +8,7 @@
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
-PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
-3.5.3.
+PyPy3.5 includes the upstream stdlib version 3.5.3.
 
 We continue to make incremental improvements to our C-API
 compatibility layer (cpyext). PyPy2 can now import and run many C-extension
@@ -19,13 +18,13 @@
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
 release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
 as `these benchmarks show`_ it already gives a nice speed bump.
 We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
-CFFI_ has been updated to 1.10, improving an already great package for
+CFFI_ has been updated to 1.11, improving an already great package for
 interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
@@ -81,48 +80,117 @@
 
 See also issues that were resolved_
 
+Note that these are also merged into PyPy 3.5
+
 * New features and cleanups
 
-  * 
-  * 
+  * Implement PyModule_New, Py_GetRecursionLimit, Py_SetRecursionLimit,
+    Py_EnterRecursiveCall, Py_LeaveRecursiveCall, populate tp_descr_get and
+    tp_descr_set slots,
+    add conversions of ``__len__``, ``__setitem__``, ``__delitem__`` to
+    appropriate C-API slots
+  * Fix for multiple inheritance in app-level for C-API defined classes
+  * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
+  * Document more differences with CPython here_
+  * Add native PyPy support to profile frames in vmprof
+  * Fix an issue with Exception order on failed import
+  * Fix for a corner case of __future__ imports
+  * Update packaged Windows zlib, sqlite, expat and OpenSSL to versions used
+    by CPython
+  * Allow windows builds to use ``jom.exe`` for compiling in parallel
+  * Rewrite ``itertools.groupby()``, following CPython
+  * Backport changes from PyPy 3.5 to minimize the code differences
+  * Improve support for BSD using patches contributed by downstream
+  * Support profile-guided optimization, enabled with --profopt, , and
+    specify training data ``profoptpath``
 
-* Bug Fixes
+* Bug Fixes 
 
-  * 
-  * 
+  * Correctly handle dict.pop where the popping key is not the same type as the
+    dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
+  * Improve our file's universal newline .readline implementation for
+    ``\n``, ``\r`` confusion
+  * Tweak issue where ctype array ``_base`` was set on empty arrays, now it
+    is closer to the implementation in CPython
+  * Fix critical bugs in shadowstack that crashed multithreaded programs and
+    very rarely showed up even in single threaded programs
+  * Remove flaky fastpath function call from ctypes
+  * Support passing a buffersize of 0 to socket.getsockopt
+  * Avoid hash() returning -1 in cpyext
 
 * Performance improvements:
 
-  * 
-  * 
+  * Tweaks made to improve performance by reducing the number of guards
+    inserted in jitted code, based on feedback from users
+  * Add garbage collector memory pressure to some c-level allocations
+  * Speed up struck.pack, struck.pack_into
+  * Performance tweaks to round(x, n) for the case n == 0
+  * Improve zipfile performance by not doing repeated string concatenation
 
 * RPython improvements
 
-  * 
-  * 
+  * Improve the default shadowstack garbage collector, fixing a crash with
+    multithreaded code and other issues
+  * Make sure lstrip consumes the entire string
+  * Support posix_fallocate and posix_fadvise, expose them on PyPy3.5
+  * Test and fix for int_and() propagating wrong bounds
+  * Improve the generated machine code by tracking the (constant) value of
+    r11 across intructions.  This lets us avoid reloading r11 with another
+    (apparently slowish) "movabs" instruction, replacing it with either
+    nothing or a cheaper variant.
+  * Performance tweaks in the x86 JIT-generated machine code: rarely taken
+    blocks are moved off-line.  Also, the temporary register used to contain
+    large constants is reused across instructions. This helps CPUs branch
+    predictor
+  * Refactor rpython.rtyper.controllerentry to use use ``@specialize`` instead
+    of ``._annspecialcase_``
+  * Refactor handling of buffers and memoryviews. Memoryviews will now be
+    accepted in a few more places, e.g. in compile()
 
 
+.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+
 Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
 ======================================================================
 
 * New features
 
-  * 
-  * 
+  * Implement main part of PEP 489 (multi-phase extension module initialization)
+  * Add docstrings to various modules and functions
+  * Adapt many CPython bug/feature fixes from CPython 3.5 to PyPy3.5
+  * Translation succeeds on Mac OS X, unfortunately our buildbot slave cannot
+    be updated to the proper development versions of OpenSSL to properly
+    package a release.
+  * Implement `` _SSLSocket.server_side``
+  * Do not silently ignore ``_swappedbytes_`` in ctypes. We now raise a
+    ``NotImplementedError``
+  * Implement and expose ``msvcrt.SetErrorMode``
+  * Implement ``PyModule_GetState``
 
 * Bug Fixes
 
-  * 
-  * 
+  * Fix inconsistencies in the xml.etree.ElementTree.Element class, which on
+    CPython is hidden by the C version from '_elementree'.
+  * OSError(None,None) is different from OSError()
+  * Get closer to supporting 32 bit windows, translation now succeeds and most
+    lib-python/3/test runs
+  * Call ``sys.__interactivehook__`` at startup
 
 * Performance improvements:
 
-  * 
+  * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+    instead of our outdated regrverbose.py script
+  * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
+  * Avoid importing the full locale module during _io initialization, 
+    CPython change fbbf8b160e8d
+  * Avoid freezing many app-level modules at translation, avoid importing many
+    modules at startup
+  * Refactor buffers, which allows an optimization for 
+    ``bytearray()[:n].tobytes()``
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
   * PEP 442: Safe object finalization
-  * PEP 489: Multi-phase extension module initialization
 
 .. _resolved: whatsnew-pypy2-5.8.0.html
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -3,6 +3,8 @@
 ==========================
 
 .. this is a revision shortly after release-pypy2.7-v5.8.0
-.. startrev: 0c91b5f4f275
+.. startrev: 558bd00b3dd8
 
+.. branch: cffi-complex
 
+Part of the upgrade to cffi 1.11
diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
new file mode 100644
--- /dev/null
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -0,0 +1,32 @@
+=========================
+What's new in PyPy3 5.7+
+=========================
+
+.. this is the revision after release-pypy3.3-5.7.x was branched
+.. startrev: afbf09453369
+
+.. branch: mtest
+Use "<python> -m test" to run the CPython test suite, as documented by CPython,
+instead of our outdated regrverbose.py script.
+
+.. branch: win32-faulthandler
+
+Enable the 'faulthandler' module on Windows;
+this unblocks the Python test suite.
+
+.. branch: superjumbo
+
+Implement posix.posix_fallocate() and posix.posix_fadvise()
+
+.. branch: py3.5-mac-translate
+
+Fix for different posix primitives on MacOS
+
+.. branch: PyBuffer
+
+Internal refactoring of memoryviews and buffers, fixing some related
+performance issues.
+
+.. branch: jumbojet
+
+Add sched_get min/max to rposix
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.11.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,6 +91,11 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
+    def complex(self):
+        with self as ptr:
+            w_result = self.ctype.complex(ptr)
+        return w_result
+
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -405,6 +410,13 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
+    def write_raw_complex_data(self, real, imag):
+        with self as ptr:
+            halfsize = self.ctype.size >> 1
+            ptr2 = rffi.ptradd(ptr, halfsize)
+            misc.write_raw_float_data(ptr, real, halfsize)
+            misc.write_raw_float_data(ptr2, imag, halfsize)
+
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -646,6 +658,7 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
+    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,8 +105,10 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,18 +212,21 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    space = self.space
-    if self.size < 0:
-        raise oefmt(space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
+
+def _notimplemented_ffi_type(self, is_result_type, extra=''):
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s",
-                self.name, self.size, place)
+    raise oefmt(self.space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s%s",
+                self.name, self.size, place, extra)
+
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    if self.size < 0:
+        raise oefmt(self.space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
+    raise _notimplemented_ffi_type(self, is_result_type)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -260,6 +263,13 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
+def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
+    raise _notimplemented_ffi_type(self, is_result_type,
+        extra = " (the support for complex types inside libffi "
+                "is mostly missing at this point, so CFFI only "
+                "supports complex types as arguments or return "
+                "value in API-mode functions)")
+
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -276,6 +286,7 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
+W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,6 +73,14 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
+    def complex(self, cdata):
+        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
+        # calling complex(), just like <cdata 'int'> cannot be directly
+        # converted by calling float()
+        space = self.space
+        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
+                    self.name)
+
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,3 +532,51 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
+
+
+class W_CTypePrimitiveComplex(W_CTypePrimitive):
+    _attrs_ = []
+
+    def cast(self, w_ob):
+        space = self.space
+        if isinstance(w_ob, cdataobj.W_CData):
+            if not isinstance(w_ob.ctype, W_CTypePrimitive):
+                raise oefmt(space.w_TypeError,
+                            "cannot cast ctype '%s' to ctype '%s'",
+                            w_ob.ctype.name, self.name)
+            w_ob = w_ob.convert_to_object()
+        #
+        imag = 0.0
+        if space.isinstance_w(w_ob, space.w_bytes):
+            real = self.cast_str(w_ob)
+        elif space.isinstance_w(w_ob, space.w_unicode):
+            real = self.cast_unicode(w_ob)
+        else:
+            real, imag = space.unpackcomplex(w_ob)
+        w_cdata = cdataobj.W_CDataMem(space, self)
+        w_cdata.write_raw_complex_data(real, imag)
+        return w_cdata
+
+    def complex(self, cdata):
+        return self.convert_to_object(cdata)
+
+    def convert_to_object(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        real = misc.read_raw_float_data(cdata, halfsize)
+        imag = misc.read_raw_float_data(cdata2, halfsize)
+        return self.space.newcomplex(real, imag)
+
+    def convert_from_object(self, cdata, w_ob):
+        space = self.space
+        real, imag = space.unpackcomplex(w_ob)
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        misc.write_raw_float_data(cdata,  real, halfsize)
+        misc.write_raw_float_data(cdata2, imag, halfsize)
+
+    def nonzero(self, cdata):
+        halfsize = self.size >> 1
+        cdata2 = rffi.ptradd(cdata, halfsize)
+        return (misc.is_nonnull_float(cdata, halfsize) |
+                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
+def eptype(name, TYPE, ctypecls, rep=1):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,6 +94,9 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
+eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
+eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
+
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,6 +8,7 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
+from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -70,6 +71,8 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
+        "float _Complex",
+        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -209,7 +212,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.
+        # trampoline functions for PyPy.  (Same with complex numbers.)
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -218,11 +221,13 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
+                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
+                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
+            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
+                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    //TOK__COMPLEX,
+    TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,6 +171,7 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
+        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -613,6 +614,7 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
+    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -668,6 +670,8 @@
         break;
     }
 
+    t1complex = 0;
+
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -678,6 +682,7 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
+        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -731,9 +736,11 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
+            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -800,6 +807,13 @@
         }
         next_token(tok);
     }
+    if (tok->kind == TOK__COMPLEX)
+    {
+        if (t1complex == 0)
+            return parse_error(tok, "_Complex type combination unsupported");
+        t1 = t1complex;
+        next_token(tok);
+    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,8 +78,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -174,37 +174,56 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
-    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type("_Complex " + name)
-        assert bool(cast(p, 0))
+        p = new_primitive_type(name + " _Complex")
+        assert bool(cast(p, 0)) is False
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j))
+        assert bool(cast(p, 0j)) is False
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
+        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert float(cast(p, INF*1j)) == INF*1j
-        assert float(cast(p, -INF)) == -INF
+        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
+        assert complex(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
+            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
 
-        assert cast(p, -1.1j) != cast(p, -1.1j)
+        assert cast(p, -1.1j) == cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        assert repr(complex(cast(p, -0j))) == '-0j'
-        assert complex(cast(p, '\x09')) == 9.0
-        assert complex(cast(p, True)) == 1.0
+        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
+        assert complex(cast(p, b'\x09')) == 9.0 + 0j
+        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
+        assert complex(cast(p, True)) == 1.0 + 0j
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(cast, new_primitive_type(name), 1+2j)
-    py.test.raises(cast, new_primitive_type("int"), 1+2j)
+        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
+        #
+        for basetype in ["char", "int", "uint64_t", "float",
+                         "double", "long double"]:
+            baseobj = cast(new_primitive_type(basetype), 65)
+            py.test.raises(TypeError, complex, baseobj)
+        #
+        BArray = new_array_type(new_pointer_type(p), 10)
+        x = newp(BArray, None)
+        x[5] = 12.34 + 56.78j
+        assert type(x[5]) is complex
+        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
+        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
+        #
+        class Foo:
+            def __complex__(self):
+                return 2 + 3j
+        assert complex(Foo()) == 2 + 3j
+        assert complex(cast(p, Foo())) == 2 + 3j
+    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1105,6 +1124,34 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
+def test_call_function_24():
+    BFloat = new_primitive_type("float")
+    BFloatComplex = new_primitive_type("float _Complex")
+    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(24))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
+def test_call_function_25():
+    BDouble = new_primitive_type("double")
+    BDoubleComplex = new_primitive_type("double _Complex")
+    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
+    if 0:   # libffi returning nonsense silently, so logic disabled for now
+        f = cast(BFunc3, _testfunc(25))
+        result = f(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
+    else:
+        f = cast(BFunc3, _testfunc(9))
+        py.test.raises(NotImplementedError, f, 12.3, 34.5)
+
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
@@ -3796,7 +3843,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,6 +148,8 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
+            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
+            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -273,6 +275,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,6 +1819,68 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
+    def test_function_returns_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float _Complex f1(float a, float b);",
+            "test_function_returns_float_complex", """
+            #include <complex.h>
+            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+    def test_function_returns_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double _Complex f1(double a, double b);",
+            "test_function_returns_double_complex", """
+            #include <complex.h>
+            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+        """, min_version=(1, 11, 0))
+        result = lib.f1(1.25, 5.1)
+        assert type(result) == complex
+        assert result.real == 1.25   # exact
+        assert result.imag == 2*5.1  # exact
+
+    def test_function_argument_float_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "float f1(float _Complex x);",
+            "test_function_argument_float_complex", """
+            #include <complex.h>
+            static float f1(float _Complex x) { return cabsf(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-5
+        result2 = lib.f1(ffi.cast("float _Complex", x))
+        assert result2 == result
+
+    def test_function_argument_double_complex(self):
+        import sys
+        if sys.platform == 'win32':
+            skip("MSVC may not support _Complex")
+        ffi, lib = self.prepare(
+            "double f1(double _Complex);",
+            "test_function_argument_double_complex", """
+            #include <complex.h>
+            static double f1(double _Complex x) { return cabs(x); }
+        """, min_version=(1, 11, 0))
+        x = complex(12.34, 56.78)
+        result = lib.f1(x)
+        assert abs(result - abs(x)) < 1e-11
+        result2 = lib.f1(ffi.cast("double _Complex", x))
+        assert result2 == result
+
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,6 +8,7 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
+from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -83,8 +84,9 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
-                return w_result_cdata.structobj
+                ctyperesptr = w_result_cdata.ctype
+                assert isinstance(ctyperesptr, W_CTypePointer)
+                return w_result_cdata._do_getitem(ctyperesptr, 0)
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -109,13 +111,14 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
+    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union>
+        farg = fargs[i]      # <ptr to struct/union/complex>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,15 +240,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,6 +1705,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
+            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,6 +156,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -281,6 +283,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,7 +48,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,7 +1553,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2001,6 +2002,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,15 +220,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Thu Jun  1 11:46:04 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 01 Jun 2017 08:46:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test: Python3 rounds to even
Message-ID: <593036bc.e412190a.a3d84.9983@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91483:4048a8bdc257
Date: 2017-06-01 16:45 +0100
http://bitbucket.org/pypy/pypy/changeset/4048a8bdc257/

Log:	Fix test: Python3 rounds to even

diff --git a/pypy/module/__builtin__/test/test_builtin.py b/pypy/module/__builtin__/test/test_builtin.py
--- a/pypy/module/__builtin__/test/test_builtin.py
+++ b/pypy/module/__builtin__/test/test_builtin.py
@@ -642,7 +642,7 @@
         assert round(-(5e15-1)) == -(5e15-1)
         assert round(-5e15) == -5e15
         assert round(5e15/2) == 5e15/2
-        assert round((5e15+1)/2) == 5e15/2+1
+        assert round((5e15+1)/2) == 5e15/2
         assert round((5e15-1)/2) == 5e15/2
         #
         inf = 1e200 * 1e200
@@ -656,7 +656,7 @@
         assert round(56294995342131.5, 3) == 56294995342131.5
         #
         for i in range(-10, 10):
-            expected = i if i < 0 else i + 1
+            expected = i + (i % 2)
             assert round(i + 0.5) == round(i + 0.5, 0) == expected
             x = i * 10 + 5
             assert round(x, -1) == round(float(x), -1) == expected * 10

From pypy.commits at gmail.com  Fri Jun  2 03:19:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 00:19:37 -0700 (PDT)
Subject: [pypy-commit] cffi char16_char32_t: ready to merge
Message-ID: <59311189.0a082e0a.3b98e.bc2c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: char16_char32_t
Changeset: r2961:68b58f5621f8
Date: 2017-06-02 08:44 +0200
http://bitbucket.org/cffi/cffi/changeset/68b58f5621f8/

Log:	ready to merge


From pypy.commits at gmail.com  Fri Jun  2 03:19:40 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 00:19:40 -0700 (PDT)
Subject: [pypy-commit] cffi default: hg merge char16_char32_t
Message-ID: <5931118c.861a190a.88d0b.6574@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2962:719b689e9e4b
Date: 2017-06-02 08:50 +0200
http://bitbucket.org/cffi/cffi/changeset/719b689e9e4b/

Log:	hg merge char16_char32_t

	Issue #315: add 'char16_t' and 'char32_t', which are explicitly-
	sized vresions of 'wchar_t'. Like the latter, it converts to unicode
	characters and arrays of it to unicode strings.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -118,7 +118,7 @@
 /* base type flag: exactly one of the following: */
 #define CT_PRIMITIVE_SIGNED   0x001   /* signed integer */
 #define CT_PRIMITIVE_UNSIGNED 0x002   /* unsigned integer */
-#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t */
+#define CT_PRIMITIVE_CHAR     0x004   /* char, wchar_t, charN_t */
 #define CT_PRIMITIVE_FLOAT    0x008   /* float, double, long double */
 #define CT_POINTER            0x010   /* pointer, excluding ptr-to-func */
 #define CT_ARRAY              0x020   /* array */
@@ -285,9 +285,7 @@
 # include "file_emulator.h"
 #endif
 
-#ifdef HAVE_WCHAR_H
-# include "wchar_helper.h"
-#endif
+#include "wchar_helper.h"
 
 typedef struct _cffi_allocator_s {
     PyObject *ca_alloc, *ca_free;
@@ -1049,12 +1047,14 @@
     }
     else if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(data, ct->ct_size)*/
-        if (ct->ct_size == sizeof(char))
+        switch (ct->ct_size) {
+        case sizeof(char):
             return PyBytes_FromStringAndSize(data, 1);
-#ifdef HAVE_WCHAR_H
-        else
-            return _my_PyUnicode_FromWideChar((wchar_t *)data, 1);
-#endif
+        case 2:
+            return _my_PyUnicode_FromChar16((cffi_char16_t *)data, 1);
+        case 4:
+            return _my_PyUnicode_FromChar32((cffi_char32_t *)data, 1);
+        }
     }
     else if (ct->ct_flags & CT_PRIMITIVE_COMPLEX) {
         Py_complex value = read_raw_complex_data(data, ct->ct_size);
@@ -1133,27 +1133,53 @@
     return -1;
 }
 
-#ifdef HAVE_WCHAR_H
-static wchar_t _convert_to_wchar_t(PyObject *init)
-{
+static cffi_char16_t _convert_to_char16_t(PyObject *init)
+{
+    char err_got[80];
+    err_got[0] = 0;
+
     if (PyUnicode_Check(init)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(init, &ordinal) == 0)
+        cffi_char16_t ordinal;
+        if (_my_PyUnicode_AsSingleChar16(init, &ordinal, err_got) == 0)
             return ordinal;
     }
     if (CData_Check(init) &&
            (((CDataObject *)init)->c_type->ct_flags & CT_PRIMITIVE_CHAR) &&
-           (((CDataObject *)init)->c_type->ct_size == sizeof(wchar_t))) {
+           (((CDataObject *)init)->c_type->ct_size == 2)) {
         char *data = ((CDataObject *)init)->c_data;
-        /*READ(data, sizeof(wchar_t))*/
-        return *(wchar_t *)data;
+        /*READ(data, 2)*/
+        return *(cffi_char16_t *)data;
     }
     PyErr_Format(PyExc_TypeError,
-                 "initializer for ctype 'wchar_t' must be a unicode string "
-                 "of length 1, not %.200s", Py_TYPE(init)->tp_name);
-    return (wchar_t)-1;
-}
-#endif
+                 "initializer for ctype 'char16_t' must be a unicode string "
+                 "of length 1, not %.200s",
+                 err_got[0] == 0 ? Py_TYPE(init)->tp_name : err_got);
+    return (cffi_char16_t)-1;
+}
+
+static cffi_char32_t _convert_to_char32_t(PyObject *init)
+{
+    char err_got[80];
+    err_got[0] = 0;
+
+    if (PyUnicode_Check(init)) {
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(init, &ordinal, err_got) == 0)
+            return ordinal;
+    }
+    if (CData_Check(init) &&
+           (((CDataObject *)init)->c_type->ct_flags & CT_PRIMITIVE_CHAR) &&
+           (((CDataObject *)init)->c_type->ct_size == 4)) {
+        char *data = ((CDataObject *)init)->c_data;
+        /*READ(data, 4)*/
+        return *(cffi_char32_t *)data;
+    }
+    PyErr_Format(PyExc_TypeError,
+                 "initializer for ctype 'char32_t' must be a unicode string "
+                 "of length 1, not %.200s",
+                 err_got[0] == 0 ? Py_TYPE(init)->tp_name : err_got);
+    return (cffi_char32_t)-1;
+}
 
 static int _convert_error(PyObject *init, const char *ct_name,
                           const char *expected)
@@ -1191,7 +1217,7 @@
 convert_from_object_bitfield(char *data, CFieldObject *cf, PyObject *init);
 
 static Py_ssize_t
-get_new_array_length(PyObject **pvalue)
+get_new_array_length(CTypeDescrObject *ctitem, PyObject **pvalue)
 {
     PyObject *value = *pvalue;
 
@@ -1204,7 +1230,12 @@
     }
     else if (PyUnicode_Check(value)) {
         /* from a unicode, we add the null terminator */
-        return _my_PyUnicode_SizeAsWideChar(value) + 1;
+        int length;
+        if (ctitem->ct_size == 2)
+            length = _my_PyUnicode_SizeAsChar16(value);
+        else
+            length = _my_PyUnicode_SizeAsChar32(value);
+        return length + 1;
     }
     else {
         Py_ssize_t explicitlength;
@@ -1235,7 +1266,8 @@
 {
     /* a special case for var-sized C99 arrays */
     if ((cf->cf_type->ct_flags & CT_ARRAY) && cf->cf_type->ct_size < 0) {
-        Py_ssize_t varsizelength = get_new_array_length(&value);
+        Py_ssize_t varsizelength = get_new_array_length(
+                                      cf->cf_type->ct_itemdescr, &value);
         if (varsizelength < 0)
             return -1;
         if (optvarsize != NULL) {
@@ -1336,14 +1368,18 @@
             memcpy(data, srcdata, n);
             return 0;
         }
-#ifdef HAVE_WCHAR_H
         else {
             Py_ssize_t n;
             if (!PyUnicode_Check(init)) {
                 expected = "unicode or list or tuple";
                 goto cannot_convert;
             }
-            n = _my_PyUnicode_SizeAsWideChar(init);
+
+            if (ctitem->ct_size == 4)
+                n = _my_PyUnicode_SizeAsChar32(init);
+            else
+                n = _my_PyUnicode_SizeAsChar16(init);
+
             if (ct->ct_length >= 0 && n > ct->ct_length) {
                 PyErr_Format(PyExc_IndexError,
                              "initializer unicode is too long for '%s' "
@@ -1352,10 +1388,12 @@
             }
             if (n != ct->ct_length)
                 n++;
-            _my_PyUnicode_AsWideChar(init, (wchar_t *)data, n);
+            if (ctitem->ct_size == 4)
+                _my_PyUnicode_AsChar32(init, (cffi_char32_t *)data, n);
+            else
+                _my_PyUnicode_AsChar16(init, (cffi_char16_t *)data, n);
             return 0;
         }
-#endif
     }
     else {
         expected = "list or tuple";
@@ -1537,22 +1575,29 @@
         return 0;
     }
     if (ct->ct_flags & CT_PRIMITIVE_CHAR) {
-        if (ct->ct_size == sizeof(char)) {
+        switch (ct->ct_size) {
+        case sizeof(char): {
             int res = _convert_to_char(init);
             if (res < 0)
                 return -1;
             data[0] = res;
             return 0;
         }
-#ifdef HAVE_WCHAR_H
-        else {
-            wchar_t res = _convert_to_wchar_t(init);
-            if (res == (wchar_t)-1 && PyErr_Occurred())
+        case 2: {
+            cffi_char16_t res = _convert_to_char16_t(init);
+            if (res == (cffi_char16_t)-1 && PyErr_Occurred())
                 return -1;
-            *(wchar_t *)data = res;
+            *(cffi_char16_t *)data = res;
             return 0;
         }
-#endif
+        case 4: {
+            int res = _convert_to_char32_t(init);
+            if (res == -1 && PyErr_Occurred())
+                return -1;
+            *(cffi_char32_t *)data = res;
+            return 0;
+        }
+        }
     }
     if (ct->ct_flags & (CT_STRUCT|CT_UNION)) {
 
@@ -2033,12 +2078,16 @@
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
         /*READ(cd->c_data, cd->c_type->ct_size)*/
-        if (cd->c_type->ct_size == sizeof(char))
+        switch (cd->c_type->ct_size) {
+        case sizeof(char):
             return PyInt_FromLong((unsigned char)cd->c_data[0]);
-#ifdef HAVE_WCHAR_H
-        else
-            return PyInt_FromLong((long)*(wchar_t *)cd->c_data);
-#endif
+        case 2:
+            return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data);
+        case 4:
+            /* NB. cast via int32_t instead of cffi_char32_t, so that
+               we expose a signed result to the user */
+            return PyInt_FromLong((long)*(int32_t *)cd->c_data);
+        }
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
         PyObject *o = cdata_float(cd);
@@ -2730,7 +2779,11 @@
     }
     else if (PyUnicode_Check(init)) {
         /* from a unicode, we add the null terminator */
-        length = _my_PyUnicode_SizeAsWideChar(init) + 1;
+        if (ctitem->ct_size == 2)
+            length = _my_PyUnicode_SizeAsChar16(init);
+        else
+            length = _my_PyUnicode_SizeAsChar32(init);
+        length += 1;
     }
     else if ((ctitem->ct_flags & CT_IS_FILE) && PyFile_Check(init)) {
         *output_data = (char *)PyFile_AsFile(init);
@@ -3456,7 +3509,7 @@
         dataoffset = offsetof(CDataObject_own_nolength, alignment);
         datasize = ct->ct_size;
         if (datasize < 0) {
-            explicitlength = get_new_array_length(&init);
+            explicitlength = get_new_array_length(ct->ct_itemdescr, &init);
             if (explicitlength < 0)
                 return NULL;
             ctitem = ct->ct_itemdescr;
@@ -3626,18 +3679,17 @@
         value = (unsigned char)PyString_AS_STRING(ob)[0];
     }
 #endif
-#ifdef HAVE_WCHAR_H
     else if (PyUnicode_Check(ob)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(ob, &ordinal) < 0) {
+        char err_buf[80];
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(ob, &ordinal, err_buf) < 0) {
             PyErr_Format(PyExc_TypeError,
-                      "cannot cast unicode string of length %zd to ctype '%s'",
-                         PyUnicode_GET_SIZE(ob), ct->ct_name);
+                         "cannot cast %s to ctype '%s'", err_buf, ct->ct_name);
             return NULL;
         }
-        value = (long)ordinal;
-    }
-#endif
+        /* the user sees char32_t being signed, but not char16_t */
+        value = (int32_t)ordinal;
+    }
     else if (PyBytes_Check(ob)) {
         int res = _convert_to_char(ob);
         if (res < 0)
@@ -3674,17 +3726,16 @@
         *out_value = (unsigned char)PyBytes_AS_STRING(io)[0];
         return 1;
     }
-#if HAVE_WCHAR_H
     else if (PyUnicode_Check(io)) {
-        wchar_t ordinal;
-        if (_my_PyUnicode_AsSingleWideChar(io, &ordinal) < 0) {
+        char ignored[80];
+        cffi_char32_t ordinal;
+        if (_my_PyUnicode_AsSingleChar32(io, &ordinal, ignored) < 0) {
             Py_DECREF(io);
             return -1;
         }
-        *out_value = (long)ordinal;
+        *out_value = (int32_t)ordinal;
         return 1;
     }
-#endif
     return 0;
 }
 
@@ -4106,6 +4157,8 @@
        EPTYPE2(fc, "float _Complex", cffi_float_complex_t, CT_PRIMITIVE_COMPLEX ) \
        EPTYPE2(dc, "double _Complex", cffi_double_complex_t, CT_PRIMITIVE_COMPLEX ) \
        ENUM_PRIMITIVE_TYPES_WCHAR                               \
+       EPTYPE2(c16, "char16_t", cffi_char16_t, CT_PRIMITIVE_CHAR ) \
+       EPTYPE2(c32, "char32_t", cffi_char32_t, CT_PRIMITIVE_CHAR ) \
        EPTYPE(b, _Bool, CT_PRIMITIVE_UNSIGNED | CT_IS_BOOL )    \
      /* the following types are not primitive in the C sense */ \
        EPTYPE(i8, int8_t, CT_PRIMITIVE_SIGNED)                  \
@@ -6036,27 +6089,46 @@
             }
             return PyBytes_FromStringAndSize(start, length);
         }
-#ifdef HAVE_WCHAR_H
         else if (cd->c_type->ct_itemdescr->ct_flags & CT_PRIMITIVE_CHAR) {
-            const wchar_t *start = (wchar_t *)cd->c_data;
-            assert(cd->c_type->ct_itemdescr->ct_size == sizeof(wchar_t));
-            if (length < 0) {
-                /*READ(start, sizeof(wchar_t))*/
-                length = 0;
-                while (start[length])
-                    length++;
-                /*READ(start, sizeof(wchar_t) * length)*/
+            switch (cd->c_type->ct_itemdescr->ct_size) {
+            case 2: {
+                const cffi_char16_t *start = (cffi_char16_t *)cd->c_data;
+                if (length < 0) {
+                    /*READ(start, 2)*/
+                    length = 0;
+                    while (start[length])
+                        length++;
+                    /*READ(start, 2 * length)*/
+                }
+                else {
+                    /*READ(start, 2 * length)*/
+                    maxlen = length;
+                    length = 0;
+                    while (length < maxlen && start[length])
+                        length++;
+                }
+                return _my_PyUnicode_FromChar16(start, length);
             }
-            else {
-                /*READ(start, sizeof(wchar_t) * length)*/
-                maxlen = length;
-                length = 0;
-                while (length < maxlen && start[length])
-                    length++;
+            case 4: {
+                const cffi_char32_t *start = (cffi_char32_t *)cd->c_data;
+                if (length < 0) {
+                    /*READ(start, 4)*/
+                    length = 0;
+                    while (start[length])
+                        length++;
+                    /*READ(start, 4 * length)*/
+                }
+                else {
+                    /*READ(start, 4 * length)*/
+                    maxlen = length;
+                    length = 0;
+                    while (length < maxlen && start[length])
+                        length++;
+                }
+                return _my_PyUnicode_FromChar32(start, length);
             }
-            return _my_PyUnicode_FromWideChar(start, length);
-        }
-#endif
+            }
+        }
     }
     else if (cd->c_type->ct_flags & CT_IS_ENUM) {
         return convert_cdata_to_enum_string(cd, 0);
@@ -6070,12 +6142,14 @@
         /*READ(cd->c_data, cd->c_type->ct_size)*/
         if (cd->c_type->ct_size == sizeof(char))
             return PyBytes_FromStringAndSize(cd->c_data, 1);
-#ifdef HAVE_WCHAR_H
         else if (cd->c_type->ct_flags & CT_PRIMITIVE_CHAR) {
-            assert(cd->c_type->ct_size == sizeof(wchar_t));
-            return _my_PyUnicode_FromWideChar((wchar_t *)cd->c_data, 1);
-        }
-#endif
+            switch (cd->c_type->ct_size) {
+            case 2:
+                return _my_PyUnicode_FromChar16((cffi_char16_t *)cd->c_data, 1);
+            case 4:
+                return _my_PyUnicode_FromChar32((cffi_char32_t *)cd->c_data, 1);
+            }
+        }
     }
     PyErr_Format(PyExc_TypeError, "string(): unexpected cdata '%s' argument",
                  cd->c_type->ct_name);
@@ -6120,12 +6194,14 @@
     /* byte- and unicode strings */
     ctitem = cd->c_type->ct_itemdescr;
     if (ctitem->ct_flags & CT_PRIMITIVE_CHAR) {
-        if (ctitem->ct_size == sizeof(char))
+        switch (ctitem->ct_size) {
+        case sizeof(char):
             return PyBytes_FromStringAndSize(cd->c_data, length);
-#ifdef HAVE_WCHAR_H
-        else if (ctitem->ct_size == sizeof(wchar_t))
-            return _my_PyUnicode_FromWideChar((wchar_t *)cd->c_data, length);
-#endif
+        case 2:
+            return _my_PyUnicode_FromChar16((cffi_char16_t *)cd->c_data,length);
+        case 4:
+            return _my_PyUnicode_FromChar32((cffi_char32_t *)cd->c_data,length);
+        }
     }
 
     /* else, the result is a list.  This implementation should be
@@ -6992,12 +7068,51 @@
     return PyBytes_FromStringAndSize(&x, 1);
 }
 
+/* backward-compatibility hack: instead of _cffi_to_c_char16_t() and
+ * _cffi_to_c_char32_t(), we have _cffi_to_c_wchar_t() handling whatever
+ * size is wchar_t, and _cffi_to_c_wchar3216_t() handling the opposite.
+ */
 #ifdef HAVE_WCHAR_H
-static PyObject *_cffi_from_c_wchar_t(wchar_t x) {
-    return _my_PyUnicode_FromWideChar(&x, 1);
-}
+typedef wchar_t cffi_wchar_t;
+#else
+typedef uint16_t cffi_wchar_t;   /* random pick... */
 #endif
 
+static cffi_wchar_t _cffi_to_c_wchar_t(PyObject *init)
+{
+    if (sizeof(cffi_wchar_t) == 2)
+        return (cffi_wchar_t)_convert_to_char16_t(init);
+    else
+        return (cffi_wchar_t)_convert_to_char32_t(init);
+}
+static PyObject *_cffi_from_c_wchar_t(cffi_wchar_t x) {
+    if (sizeof(cffi_wchar_t) == 2) {
+        cffi_char16_t input = x;
+        return _my_PyUnicode_FromChar16(&input, 1);
+    }
+    else {
+        cffi_char32_t input = x;
+        return _my_PyUnicode_FromChar32(&input, 1);
+    }
+}
+static int _cffi_to_c_wchar3216_t(PyObject *init)
+{
+    if (sizeof(cffi_wchar_t) == 4)
+        return (int)_convert_to_char16_t(init);
+    else
+        return (int)_convert_to_char32_t(init);
+}
+static PyObject *_cffi_from_c_wchar3216_t(int x) {
+    if (sizeof(cffi_wchar_t) == 4) {
+        cffi_char16_t input = x;
+        return _my_PyUnicode_FromChar16(&input, 1);
+    }
+    else {
+        cffi_char32_t input = x;
+        return _my_PyUnicode_FromChar32(&input, 1);
+    }
+}
+
 struct _cffi_externpy_s;      /* forward declaration */
 static void cffi_call_python(struct _cffi_externpy_s *, char *args);
 
@@ -7021,18 +7136,15 @@
     convert_to_object,
     convert_from_object,
     convert_struct_to_owning_object,
-#ifdef HAVE_WCHAR_H
-    _convert_to_wchar_t,
+    _cffi_to_c_wchar_t,
     _cffi_from_c_wchar_t,
-#else
-    0,
-    0,
-#endif
     _cffi_to_c_long_double,
     _cffi_to_c__Bool,
     _prepare_pointer_call_argument,
     convert_array_from_object,
     cffi_call_python,
+    _cffi_to_c_wchar3216_t,
+    _cffi_from_c_wchar3216_t,
 };
 
 static struct { const char *name; int value; } all_dlopen_flags[] = {
diff --git a/c/cffi1_module.c b/c/cffi1_module.c
--- a/c/cffi1_module.c
+++ b/c/cffi1_module.c
@@ -2,8 +2,9 @@
 #include "parse_c_type.c"
 #include "realize_c_type.c"
 
-#define CFFI_VERSION_MIN    0x2601
-#define CFFI_VERSION_MAX    0x27FF
+#define CFFI_VERSION_MIN            0x2601
+#define CFFI_VERSION_CHAR16CHAR32   0x2801
+#define CFFI_VERSION_MAX            0x28FF
 
 typedef struct FFIObject_s FFIObject;
 typedef struct LibObject_s LibObject;
@@ -183,6 +184,8 @@
     num_exports = 25;
     if (ctx->flags & 1)    /* set to mean that 'extern "Python"' is used */
         num_exports = 26;
+    if (version >= CFFI_VERSION_CHAR16CHAR32)
+        num_exports = 28;
     memcpy(exports, (char *)cffi_exports, num_exports * sizeof(void *));
 
     /* make the module object */
diff --git a/c/parse_c_type.c b/c/parse_c_type.c
--- a/c/parse_c_type.c
+++ b/c/parse_c_type.c
@@ -493,6 +493,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -501,6 +502,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -153,6 +153,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
     };
     PyObject *x;
 
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2098,22 +2098,36 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+
+def test_char16():
+    assert sizeof(new_primitive_type("char16_t")) == 2
+    _test_wchar_variant("char16_t")
+
+def test_char32():
+    assert sizeof(new_primitive_type("char32_t")) == 4
+    _test_wchar_variant("char32_t")
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2128,9 +2142,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2165,17 +2179,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2183,8 +2197,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2211,7 +2225,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)
diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -2,31 +2,28 @@
  * wchar_t helpers
  */
 
-#if (Py_UNICODE_SIZE == 2) && (SIZEOF_WCHAR_T == 4)
-# define CONVERT_WCHAR_TO_SURROGATES
-#endif
+typedef uint16_t cffi_char16_t;
+typedef uint32_t cffi_char32_t;
+/* NB. cffi_char32_t is unsigned to make the logic here a bit easier */
 
 
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 2
 
 /* Before Python 2.7, PyUnicode_FromWideChar is not able to convert
    wchar_t values greater than 65535 into two-unicode-characters surrogates.
    But even the Python 2.7 version doesn't detect wchar_t values that are
    out of range(1114112), and just returns nonsense.
+
+   From cffi 1.11 we can't use it anyway, because we need a version
+   with char32_t input types.
 */
 static PyObject *
-_my_PyUnicode_FromWideChar(register const wchar_t *w,
-                           Py_ssize_t size)
+_my_PyUnicode_FromChar32(const cffi_char32_t *w, Py_ssize_t size)
 {
     PyObject *unicode;
     register Py_ssize_t i;
     Py_ssize_t alloc;
-    const wchar_t *orig_w;
-
-    if (w == NULL) {
-        PyErr_BadInternalCall();
-        return NULL;
-    }
+    const cffi_char32_t *orig_w;
 
     alloc = size;
     orig_w = w;
@@ -45,11 +42,11 @@
         register Py_UNICODE *u;
         u = PyUnicode_AS_UNICODE(unicode);
         for (i = size; i > 0; i--) {
-            if (((unsigned int)*w) > 0xFFFF) {
-                wchar_t ordinal;
-                if (((unsigned int)*w) > 0x10FFFF) {
+            if (*w > 0xFFFF) {
+                cffi_char32_t ordinal;
+                if (*w > 0x10FFFF) {
                     PyErr_Format(PyExc_ValueError,
-                                 "wchar_t out of range for "
+                                 "char32_t out of range for "
                                  "conversion to unicode: 0x%x", (int)*w);
                     Py_DECREF(unicode);
                     return NULL;
@@ -66,9 +63,55 @@
     return unicode;
 }
 
-#else
+static PyObject *
+_my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
+{
+    return PyUnicode_FromUnicode((const Py_UNICODE *)w, size);
+}
 
-# define _my_PyUnicode_FromWideChar PyUnicode_FromWideChar
+#else   /* Py_UNICODE_SIZE == 4 */
+
+static PyObject *
+_my_PyUnicode_FromChar32(const cffi_char32_t *w, Py_ssize_t size)
+{
+    return PyUnicode_FromUnicode((const Py_UNICODE *)w, size);
+}
+
+static PyObject *
+_my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
+{
+    /* 'size' is the length of the 'w' array */
+    PyObject *result = PyUnicode_FromUnicode(NULL, size);
+
+    if (result != NULL) {
+        Py_UNICODE *u_base = PyUnicode_AS_UNICODE(result);
+        Py_UNICODE *u = u_base;
+
+        if (size == 1) {      /* performance only */
+            *u = (cffi_char32_t)*w;
+        }
+        else {
+            while (size > 0) {
+                cffi_char32_t ch = *w++;
+                size--;
+                if (0xD800 <= ch && ch <= 0xDBFF && size > 0) {
+                    cffi_char32_t ch2 = *w;
+                    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
+                        ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+                        w++;
+                        size--;
+                    }
+                }
+                *u++ = ch;
+            }
+            if (PyUnicode_Resize(&result, u - u_base) < 0) {
+                Py_DECREF(result);
+                return NULL;
+            }
+        }
+    }
+    return result;
+}
 
 #endif
 
@@ -78,28 +121,70 @@
 #define AS_SURROGATE(u)   (0x10000 + (((u)[0] - 0xD800) << 10) +     \
                                      ((u)[1] - 0xDC00))
 
-static int _my_PyUnicode_AsSingleWideChar(PyObject *unicode, wchar_t *result)
+static int
+_my_PyUnicode_AsSingleChar16(PyObject *unicode, cffi_char16_t *result,
+                             char *err_got)
+{
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    if (PyUnicode_GET_SIZE(unicode) != 1) {
+        sprintf(err_got, "unicode string of length %zd",
+                PyUnicode_GET_SIZE(unicode));
+        return -1;
+    }
+#if Py_UNICODE_SIZE == 4
+    if (((unsigned int)u[0]) > 0xFFFF)
+    {
+        sprintf(err_got, "larger-than-0xFFFF character");
+        return -1;
+    }
+#endif
+    *result = (cffi_char16_t)u[0];
+    return 0;
+}
+
+static int
+_my_PyUnicode_AsSingleChar32(PyObject *unicode, cffi_char32_t *result,
+                             char *err_got)
 {
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     if (PyUnicode_GET_SIZE(unicode) == 1) {
-        *result = (wchar_t)(u[0]);
+        *result = (cffi_char32_t)u[0];
         return 0;
     }
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 2
     if (PyUnicode_GET_SIZE(unicode) == 2 && IS_SURROGATE(u)) {
         *result = AS_SURROGATE(u);
         return 0;
     }
 #endif
+    sprintf(err_got, "unicode string of length %zd",
+            PyUnicode_GET_SIZE(unicode));
     return -1;
 }
 
-static Py_ssize_t _my_PyUnicode_SizeAsWideChar(PyObject *unicode)
+static Py_ssize_t _my_PyUnicode_SizeAsChar16(PyObject *unicode)
 {
     Py_ssize_t length = PyUnicode_GET_SIZE(unicode);
     Py_ssize_t result = length;
 
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+#if Py_UNICODE_SIZE == 4
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    Py_ssize_t i;
+
+    for (i=0; i<length; i++) {
+        if (u[i] > 0xFFFF)
+            result++;
+    }
+#endif
+    return result;
+}
+
+static Py_ssize_t _my_PyUnicode_SizeAsChar32(PyObject *unicode)
+{
+    Py_ssize_t length = PyUnicode_GET_SIZE(unicode);
+    Py_ssize_t result = length;
+
+#if Py_UNICODE_SIZE == 2
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     Py_ssize_t i;
 
@@ -111,15 +196,41 @@
     return result;
 }
 
-static void _my_PyUnicode_AsWideChar(PyObject *unicode,
-                                     wchar_t *result,
-                                     Py_ssize_t resultlen)
+static void _my_PyUnicode_AsChar16(PyObject *unicode,
+                                   cffi_char16_t *result,
+                                   Py_ssize_t resultlen)
+{
+    Py_ssize_t len = PyUnicode_GET_SIZE(unicode);
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
+    Py_ssize_t i;
+    for (i=0; i<len; i++) {
+#if Py_UNICODE_SIZE == 2
+        cffi_char16_t ordinal = u[i];
+#else
+        cffi_char32_t ordinal = u[i];
+        if (ordinal > 0xFFFF) {
+            /* NB. like CPython, ignore the problem of unicode string objects
+             * containing characters greater than sys.maxunicode.  It is
+             * easier to not add exception handling here */
+            ordinal -= 0x10000;
+            *result++ = 0xD800 | (ordinal >> 10);
+            *result++ = 0xDC00 | (ordinal & 0x3FF);
+            continue;
+        }
+#endif
+        *result++ = ordinal;
+    }
+}
+
+static void _my_PyUnicode_AsChar32(PyObject *unicode,
+                                   cffi_char32_t *result,
+                                   Py_ssize_t resultlen)
 {
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     Py_ssize_t i;
     for (i=0; i<resultlen; i++) {
-        wchar_t ordinal = *u;
-#ifdef CONVERT_WCHAR_TO_SURROGATES
+        cffi_char32_t ordinal = *u;
+#if Py_UNICODE_SIZE == 2
         if (IS_SURROGATE(u)) {
             ordinal = AS_SURROGATE(u);
             u++;
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -62,11 +62,16 @@
     typedef unsigned char _Bool;
 #  endif
 # endif
+# if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
+    typedef uint16_t char16_t;
+    typedef int32_t char32_t;
+# endif
 #else
 # include <stdint.h>
 # if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
+# include <uchar.h>
 #endif
 
 #ifdef __GNUC__
@@ -159,9 +164,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +179,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +224,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/cffi/cffi_opcode.py b/cffi/cffi_opcode.py
--- a/cffi/cffi_opcode.py
+++ b/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/cffi/parse_c_type.h b/cffi/parse_c_type.h
--- a/cffi/parse_c_type.h
+++ b/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/testing/cffi0/test_ffi_backend.py b/testing/cffi0/test_ffi_backend.py
--- a/testing/cffi0/test_ffi_backend.py
+++ b/testing/cffi0/test_ffi_backend.py
@@ -1,6 +1,7 @@
 import py, sys, platform
 import pytest
 from testing.cffi0 import backend_tests, test_function, test_ownlib
+from testing.support import u
 from cffi import FFI
 import _cffi_backend
 
@@ -397,6 +398,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -508,3 +511,43 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+    def test_explicitly_defined_char16_t(self):
+        ffi = FFI()
+        ffi.cdef("typedef uint16_t char16_t;")
+        x = ffi.cast("char16_t", 1234)
+        assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+    def test_char16_t(self):
+        ffi = FFI()
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        ffi = FFI()
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        py_uni = u+'\U00012345'
+        z = ffi.new("char32_t[]", py_uni)
+        assert len(z) == 2
+        assert list(z) == [py_uni, u+'\x00']    # maybe a 2-unichars string
+        assert ffi.string(z) == py_uni
+        if len(py_uni) == 1:    # 4-bytes unicodes in Python
+            s = ffi.new("char32_t[]", u+'\ud808\udf00')
+            assert len(s) == 3
+            assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/testing/cffi0/test_ownlib.py b/testing/cffi0/test_ownlib.py
--- a/testing/cffi0/test_ownlib.py
+++ b/testing/cffi0/test_ownlib.py
@@ -2,6 +2,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
+from testing.support import u
 
 
 SOURCE = """\
@@ -92,6 +93,15 @@
 }
 
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+    return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+    return (unsigned int)(a + 42);
+}
 """
 
 class TestOwnLib(object):
@@ -300,3 +310,18 @@
         pfn = ffi.addressof(lib, "test_getting_errno")
         assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
         assert pfn == lib.test_getting_errno
+
+    def test_char16_char32_t(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            char16_t foo_2bytes(char16_t);
+            char32_t foo_4bytes(char32_t);
+        """)
+        lib = ffi.dlopen(self.module)
+        assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/testing/cffi0/test_verify.py b/testing/cffi0/test_verify.py
--- a/testing/cffi0/test_verify.py
+++ b/testing/cffi0/test_verify.py
@@ -241,7 +241,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true
@@ -384,6 +384,10 @@
     lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
     assert lib.foo(uniexample1) == uniexample2
 
+def test_char16_char32_type():
+    py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+                 "working in ffi.verify() mode")
+
 def test_no_argument():
     ffi = FFI()
     ffi.cdef("int foo(void);")
diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -1672,6 +1672,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -1742,3 +1744,30 @@
         exec("from _test_import_from_lib import *", d)
         assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
                 ['ffi', 'lib'])
+
+    def test_char16_t(self):
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -24,10 +24,11 @@
     assert ''.join(map(str, recomp.cffi_types)) == expected_output
 
 def verify(ffi, module_name, source, *args, **kwds):
+    no_cpp = kwds.pop('no_cpp', False)
     kwds.setdefault('undef_macros', ['NDEBUG'])
     module_name = '_CFFI_' + module_name
     ffi.set_source(module_name, source)
-    if not os.environ.get('NO_CPP'):     # test the .cpp mode too
+    if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
     else:
@@ -2250,3 +2251,29 @@
     int f(int a) { return a + 40; }
     """, extra_compile_args=['-fvisibility=hidden'])
     assert lib.f(2) == 42
+
+def test_override_default_definition():
+    ffi = FFI()
+    ffi.cdef("typedef long int16_t, char16_t;")
+    lib = verify(ffi, "test_override_default_definition", "")
+    assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+    ffi = FFI()
+    ffi.cdef("""
+        char16_t foo_2bytes(char16_t);
+        char32_t foo_4bytes(char32_t);
+    """)
+    lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+    char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+    """, no_cpp=no_cpp)
+    assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+    py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+    py.test.raises(TypeError, lib.foo_2bytes, 1234)
+    py.test.raises(TypeError, lib.foo_4bytes, 1234)
+
+def test_char16_char32_plain_c():
+    test_char16_char32_type(no_cpp=True)
diff --git a/testing/cffi1/test_verify1.py b/testing/cffi1/test_verify1.py
--- a/testing/cffi1/test_verify1.py
+++ b/testing/cffi1/test_verify1.py
@@ -221,7 +221,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true

From pypy.commits at gmail.com  Fri Jun  2 03:19:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 00:19:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document char16_t and char32_t
Message-ID: <5931118e.1196190a.61af1.c063@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2963:c60281bf502f
Date: 2017-06-02 09:19 +0200
http://bitbucket.org/cffi/cffi/changeset/c60281bf502f/

Log:	Document char16_t and char32_t

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -178,7 +178,8 @@
 * intN_t, uintN_t (for N=8,16,32,64), intptr_t, uintptr_t, ptrdiff_t,
   size_t, ssize_t
 
-* wchar_t (if supported by the backend)
+* wchar_t (if supported by the backend).  *New in version 1.11:*
+  char16_t and char32_t.
 
 * _Bool and bool (equivalent).  If not directly supported by the C
   compiler, this is declared with the size of ``unsigned char``.
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -104,11 +104,13 @@
   returns a ``bytes``, not a ``str``.
 
 - If 'cdata' is a pointer or array of wchar_t, returns a unicode string
-  following the same rules.
+  following the same rules.  *New in version 1.11:* can also be
+  char16_t or char32_t.
 
-- If 'cdata' is a single character or byte or a wchar_t, returns it as a
-  byte string or unicode string.  (Note that in some situation a single
-  wchar_t may require a Python unicode string of length 2.)
+- If 'cdata' is a single character or byte or a wchar_t or charN_t,
+  returns it as a byte string or unicode string.  (Note that in some
+  situation a single wchar_t or char32_t may require a Python unicode
+  string of length 2.)
 
 - If 'cdata' is an enum, returns the value of the enumerator as a string.
   If the value is out of range, it is simply returned as the stringified
@@ -125,7 +127,7 @@
 
 - If 'cdata' is a pointer to 'wchar_t', returns a unicode string.
   ('length' is measured in number of wchar_t; it is not the size in
-  bytes.)
+  bytes.)  *New in version 1.11:* can also be char16_t or char32_t.
 
 - If 'cdata' is a pointer to anything else, returns a list, of the
   given 'length'.  (A slower way to do that is ``[cdata[i] for i in
@@ -626,10 +628,10 @@
 |   ``char``    | a string of length 1   | a string of      | int(), bool(), |
 |               | or another <cdata char>| length 1         | ``<``          |
 +---------------+------------------------+------------------+----------------+
-|  ``wchar_t``  | a unicode of length 1  | a unicode of     |                |
-|               | (or maybe 2 if         | length 1         | int(), bool(), |
-|               | surrogates) or         | (or maybe 2 if   | ``<``          |
-|               | another <cdata wchar_t>| surrogates)      |                |
+| ``wchar_t``,  | a unicode of length 1  | a unicode of     |                |
+| ``char16_t``, | (or maybe 2 if         | length 1         | int(), bool(), |
+| ``char32_t``  | surrogates) or         | (or maybe 2 if   | ``<``          |
+|               | another similar <cdata>| surrogates)      |                |
 +---------------+------------------------+------------------+----------------+
 |  ``float``,   | a float or anything on | a Python float   | float(), int(),|
 |  ``double``   | which float() works    |                  | bool(), ``<``  |
@@ -671,9 +673,9 @@
 | ``char[]``,   |                        |                  | ``-``          |
 | ``_Bool[]``   |                        |                  |                |
 +---------------+------------------------+                  +----------------+
-| ``wchar_t[]`` | same as arrays, or a   |                  | len(), iter(), |
-|               | Python unicode string  |                  | ``[]``,        |
-|               |                        |                  | ``+``, ``-``   |
+|``wchar_t[]``, | same as arrays, or a   |                  | len(), iter(), |
+|``char16_t[]``,| Python unicode string  |                  | ``[]``,        |
+|``char32_t[]`` |                        |                  | ``+``, ``-``   |
 |               |                        |                  |                |
 +---------------+------------------------+------------------+----------------+
 | structure     | a list or tuple or     | a <cdata>        | read/write     |
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -25,6 +25,11 @@
 unicode string to an integer, ``ord(x)`` does not work; use instead
 ``int(ffi.cast('wchar_t', x))``.
 
+*New in version 1.11:* in addition to ``wchar_t``, the C types
+``char16_t`` and ``char32_t`` work the same but with a known fixed size.
+In previous versions, this could be achieved using ``uint16_t`` and
+``int32_t`` but without automatic convertion to Python unicodes.
+
 Pointers, structures and arrays are more complex: they don't have an
 obvious Python equivalent.  Thus, they correspond to objects of type
 ``cdata``, which are printed for example as
@@ -197,9 +202,10 @@
     >>> ffi.string(x) # interpret 'x' as a regular null-terminated string
     'Hello'
 
-Similarly, arrays of wchar_t can be initialized from a unicode string,
+Similarly, arrays of wchar_t or char16_t or char32_t can be initialized
+from a unicode string,
 and calling ``ffi.string()`` on the cdata object returns the current unicode
-string stored in the wchar_t array (adding surrogates if necessary).
+string stored in the source array (adding surrogates if necessary).
 
 Note that unlike Python lists or tuples, but like C, you *cannot* index in
 a C array from the end using negative numbers.
@@ -347,7 +353,8 @@
 
     assert lib.strlen("hello") == 5
 
-You can also pass unicode strings as ``wchar_t *`` arguments.  Note that
+You can also pass unicode strings as ``wchar_t *`` or ``char16_t *`` or
+``char32_t *`` arguments.  Note that
 the C language makes no difference between argument declarations that
 use ``type *`` or ``type[]``.  For example, ``int *`` is fully
 equivalent to ``int[]`` (or even ``int[5]``; the 5 is ignored).  For CFFI,
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -6,6 +6,13 @@
 v1.11
 =====
 
+* Support the modern standard types ``char16_t`` and ``char32_t``.
+  These work like ``wchar_t``: they represent one unicode character, or
+  when used as ``charN_t *`` or ``charN_t[]`` they represent a unicode
+  string.  The difference with ``wchar_t`` is that they have a known,
+  fixed size.  They should work at all places that used to work with
+  ``wchar_t`` (please report an issue if I missing something).
+
 * Support the C99 types ``float _Complex`` and ``double _Complex``.
   Note that libffi doesn't support them, which means that in the ABI
   mode you still cannot call C functions that take complex numbers

From pypy.commits at gmail.com  Fri Jun  2 03:23:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 00:23:00 -0700 (PDT)
Subject: [pypy-commit] cffi default: Don't pass '-Werror' to MSVC
Message-ID: <59311254.47b5190a.d7cbb.c324@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2964:19722530a814
Date: 2017-06-02 09:22 +0200
http://bitbucket.org/cffi/cffi/changeset/19722530a814/

Log:	Don't pass '-Werror' to MSVC

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -31,7 +31,7 @@
     if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
-    else:
+    elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])

From pypy.commits at gmail.com  Fri Jun  2 03:25:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 00:25:49 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: hg merge default
Message-ID: <593112fd.02532e0a.89afd.b90c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91484:fc36c63955e1
Date: 2017-06-02 09:25 +0200
http://bitbucket.org/pypy/pypy/changeset/fc36c63955e1/

Log:	hg merge default

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.11.0"
+__version_info__ = (1, 11, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.10.0"
+                               "\ncompiled with cffi version: 1.11.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,8 +105,11 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
+PRIM_FLOATCOMPLEX  = 48
+PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 48
+
+_NUM_PRIM          = 50
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -128,6 +131,8 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,7 +95,8 @@
 
 
 class BasePrimitiveType(BaseType):
-    pass
+    def is_complex_type(self):
+        return False
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -116,6 +117,8 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
+        'float _Complex':     'j',
+        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -163,6 +166,8 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
+    def is_complex_type(self):
+        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,8 +79,10 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
+#define _CFFI_PRIM_FLOATCOMPLEX 48
+#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         48
+#define _CFFI__NUM_PRIM         50
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType):
+        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,8 +524,10 @@
                                                     tovar, errcode)
             return
         #
-        elif isinstance(tp, model.StructOrUnionOrEnum):
-            # a struct (not a struct pointer) as a function argument
+        elif (isinstance(tp, model.StructOrUnionOrEnum) or
+              isinstance(tp, model.BasePrimitiveType)):
+            # a struct (not a struct pointer) as a function argument;
+            # or, a complex (the same code works)
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -570,7 +572,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double':
+            elif tp.name != 'long double' and not tp.is_complex_type():
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -734,21 +736,26 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.
+        # arg that is a pointer to the result.  We also do that for
+        # complex args and return type.
+        def need_indirection(type):
+            return (isinstance(type, model.StructOrUnion) or
+                    (isinstance(type, model.PrimitiveType) and
+                     type.is_complex_type()))
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if isinstance(type, model.StructOrUnion):
+            if need_indirection(type):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if isinstance(tp_result, model.StructOrUnion):
+        if need_indirection(tp_result):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1180,7 +1187,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s", %s };' % (name, size_of_result))
+        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1479,6 +1486,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -8,8 +8,7 @@
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
-PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
-3.5.3.
+PyPy3.5 includes the upstream stdlib version 3.5.3.
 
 We continue to make incremental improvements to our C-API
 compatibility layer (cpyext). PyPy2 can now import and run many C-extension
@@ -19,13 +18,13 @@
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
 release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
 as `these benchmarks show`_ it already gives a nice speed bump.
 We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
-CFFI_ has been updated to 1.10, improving an already great package for
+CFFI_ has been updated to 1.11, improving an already great package for
 interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
@@ -81,28 +80,52 @@
 
 See also issues that were resolved_
 
+Note that these are also merged into PyPy 3.5
+
 * New features and cleanups
 
-  * Implement PyModule_New, 
+  * Implement PyModule_New, Py_GetRecursionLimit, Py_SetRecursionLimit,
+    Py_EnterRecursiveCall, Py_LeaveRecursiveCall, populate tp_descr_get and
+    tp_descr_set slots,
+    add conversions of ``__len__``, ``__setitem__``, ``__delitem__`` to
+    appropriate C-API slots
   * Fix for multiple inheritance in app-level for C-API defined classes
   * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
   * Document more differences with CPython here_
   * Add native PyPy support to profile frames in vmprof
   * Fix an issue with Exception order on failed import
   * Fix for a corner case of __future__ imports
+  * Update packaged Windows zlib, sqlite, expat and OpenSSL to versions used
+    by CPython
+  * Allow windows builds to use ``jom.exe`` for compiling in parallel
+  * Rewrite ``itertools.groupby()``, following CPython
+  * Backport changes from PyPy 3.5 to minimize the code differences
+  * Improve support for BSD using patches contributed by downstream
+  * Support profile-guided optimization, enabled with --profopt, , and
+    specify training data ``profoptpath``
 
-* Bug Fixes
+* Bug Fixes 
 
   * Correctly handle dict.pop where the popping key is not the same type as the
     dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
   * Improve our file's universal newline .readline implementation for
     ``\n``, ``\r`` confusion
+  * Tweak issue where ctype array ``_base`` was set on empty arrays, now it
+    is closer to the implementation in CPython
+  * Fix critical bugs in shadowstack that crashed multithreaded programs and
+    very rarely showed up even in single threaded programs
+  * Remove flaky fastpath function call from ctypes
+  * Support passing a buffersize of 0 to socket.getsockopt
+  * Avoid hash() returning -1 in cpyext
 
 * Performance improvements:
 
   * Tweaks made to improve performance by reducing the number of guards
     inserted in jitted code, based on feedback from users
   * Add garbage collector memory pressure to some c-level allocations
+  * Speed up struck.pack, struck.pack_into
+  * Performance tweaks to round(x, n) for the case n == 0
+  * Improve zipfile performance by not doing repeated string concatenation
 
 * RPython improvements
 
@@ -119,6 +142,11 @@
     blocks are moved off-line.  Also, the temporary register used to contain
     large constants is reused across instructions. This helps CPUs branch
     predictor
+  * Refactor rpython.rtyper.controllerentry to use use ``@specialize`` instead
+    of ``._annspecialcase_``
+  * Refactor handling of buffers and memoryviews. Memoryviews will now be
+    accepted in a few more places, e.g. in compile()
+
 
 .. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
 
@@ -129,6 +157,15 @@
 
   * Implement main part of PEP 489 (multi-phase extension module initialization)
   * Add docstrings to various modules and functions
+  * Adapt many CPython bug/feature fixes from CPython 3.5 to PyPy3.5
+  * Translation succeeds on Mac OS X, unfortunately our buildbot slave cannot
+    be updated to the proper development versions of OpenSSL to properly
+    package a release.
+  * Implement `` _SSLSocket.server_side``
+  * Do not silently ignore ``_swappedbytes_`` in ctypes. We now raise a
+    ``NotImplementedError``
+  * Implement and expose ``msvcrt.SetErrorMode``
+  * Implement ``PyModule_GetState``
 
 * Bug Fixes
 
@@ -137,12 +174,19 @@
   * OSError(None,None) is different from OSError()
   * Get closer to supporting 32 bit windows, translation now succeeds and most
     lib-python/3/test runs
+  * Call ``sys.__interactivehook__`` at startup
 
 * Performance improvements:
 
   * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
     instead of our outdated regrverbose.py script
   * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
+  * Avoid importing the full locale module during _io initialization, 
+    CPython change fbbf8b160e8d
+  * Avoid freezing many app-level modules at translation, avoid importing many
+    modules at startup
+  * Refactor buffers, which allows an optimization for 
+    ``bytearray()[:n].tobytes()``
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,4 +5,6 @@
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8
 
+.. branch: cffi-complex
 
+Part of the upgrade to cffi 1.11
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,15 +240,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,6 +1705,8 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
+            'float _Complex',
+            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,6 +156,8 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
+            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
+            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -281,6 +283,11 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
+    #
+    parse_error("_Complex", "identifier expected", 0)
+    parse_error("int _Complex", "_Complex type combination unsupported", 4)
+    parse_error("long double _Complex", "_Complex type combination unsupported",
+                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,7 +48,6 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
-
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,7 +1553,8 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function bar() called, but no code was attached "
+        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
+        b"but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2001,6 +2002,60 @@
     """)
     assert lib.f1(52).a == 52
 
+def test_function_returns_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float _Complex f1(float a, float b);");
+    lib = verify(ffi, "test_function_returns_float_complex", """
+        #include <complex.h>
+        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
+
+def test_function_returns_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double _Complex f1(double a, double b);");
+    lib = verify(ffi, "test_function_returns_double_complex", """
+        #include <complex.h>
+        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
+    """)
+    result = lib.f1(1.25, 5.1)
+    assert type(result) == complex
+    assert result.real == 1.25   # exact
+    assert result.imag == 2*5.1  # exact
+
+def test_function_argument_float_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("float f1(float _Complex x);");
+    lib = verify(ffi, "test_function_argument_float_complex", """
+        #include <complex.h>
+        static float f1(float _Complex x) { return cabsf(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-5
+
+def test_function_argument_double_complex():
+    if sys.platform == 'win32':
+        py.test.skip("MSVC may not support _Complex")
+    ffi = FFI()
+    ffi.cdef("double f1(double _Complex);");
+    lib = verify(ffi, "test_function_argument_double_complex", """
+        #include <complex.h>
+        static double f1(double _Complex x) { return cabs(x); }
+    """)
+    x = complex(12.34, 56.78)
+    result = lib.f1(x)
+    assert abs(result - abs(x)) < 1e-11
+
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,15 +220,18 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
+        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert I + F + C == 1      # one and only one of them is true
+        assert X == (typename in ('float _Complex', 'double _Complex'))
+        assert I + F + C + X == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
+            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Fri Jun  2 04:02:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 01:02:47 -0700 (PDT)
Subject: [pypy-commit] cffi default: Fix: the type char32_t is defined by
 the standard as always unsigned
Message-ID: <59311ba7.051b190a.794dc.ba29@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2965:c0c2020872d3
Date: 2017-06-02 10:02 +0200
http://bitbucket.org/cffi/cffi/changeset/c0c2020872d3/

Log:	Fix: the type char32_t is defined by the standard as always unsigned

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -143,6 +143,7 @@
 #define CT_IS_UNSIZED_CHAR_A   0x00800000
 #define CT_LAZY_FIELD_LIST     0x01000000
 #define CT_WITH_PACKED_CHANGE  0x02000000
+#define CT_IS_SIGNED_WCHAR     0x04000000
 #define CT_PRIMITIVE_ANY  (CT_PRIMITIVE_SIGNED |        \
                            CT_PRIMITIVE_UNSIGNED |      \
                            CT_PRIMITIVE_CHAR |          \
@@ -1591,8 +1592,8 @@
             return 0;
         }
         case 4: {
-            int res = _convert_to_char32_t(init);
-            if (res == -1 && PyErr_Occurred())
+            cffi_char32_t res = _convert_to_char32_t(init);
+            if (res == (cffi_char32_t)-1 && PyErr_Occurred())
                 return -1;
             *(cffi_char32_t *)data = res;
             return 0;
@@ -2084,9 +2085,12 @@
         case 2:
             return PyInt_FromLong((long)*(cffi_char16_t *)cd->c_data);
         case 4:
-            /* NB. cast via int32_t instead of cffi_char32_t, so that
-               we expose a signed result to the user */
-            return PyInt_FromLong((long)*(int32_t *)cd->c_data);
+            if (cd->c_type->ct_flags & CT_IS_SIGNED_WCHAR)
+                return PyInt_FromLong((long)*(int32_t *)cd->c_data);
+            else if (sizeof(long) > 4)
+                return PyInt_FromLong(*(uint32_t *)cd->c_data);
+            else
+                return PyLong_FromUnsignedLong(*(uint32_t *)cd->c_data);
         }
     }
     else if (cd->c_type->ct_flags & CT_PRIMITIVE_FLOAT) {
@@ -3687,8 +3691,15 @@
                          "cannot cast %s to ctype '%s'", err_buf, ct->ct_name);
             return NULL;
         }
-        /* the user sees char32_t being signed, but not char16_t */
-        value = (int32_t)ordinal;
+        /* the types char16_t and char32_t are both unsigned.  However,
+           wchar_t might be signed.  In theory it does not matter,
+           because 'ordinal' comes from a regular Python unicode. */
+#ifdef HAVE_WCHAR_H
+        if (ct->ct_flags & CT_IS_SIGNED_WCHAR)
+            value = (wchar_t)ordinal;
+        else
+#endif
+            value = ordinal;
     }
     else if (PyBytes_Check(ob)) {
         int res = _convert_to_char(ob);
@@ -3733,7 +3744,10 @@
             Py_DECREF(io);
             return -1;
         }
-        *out_value = (int32_t)ordinal;
+        /* the signness of the 32-bit version of wide chars should not
+         * matter here, because 'ordinal' comes from a normal Python
+         * unicode string */
+        *out_value = ordinal;
         return 1;
     }
     return 0;
@@ -4195,7 +4209,8 @@
 
 #ifdef HAVE_WCHAR_H
 # define ENUM_PRIMITIVE_TYPES_WCHAR                             \
-       EPTYPE(wc, wchar_t, CT_PRIMITIVE_CHAR )
+       EPTYPE(wc, wchar_t, CT_PRIMITIVE_CHAR |                  \
+                           (((wchar_t)-1) > 0 ? 0 : CT_IS_SIGNED_WCHAR))
 #else
 # define ENUM_PRIMITIVE_TYPES_WCHAR   /* nothing */
 #endif
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2101,12 +2101,16 @@
     _test_wchar_variant("wchar_t")
 
 def test_char16():
-    assert sizeof(new_primitive_type("char16_t")) == 2
+    BChar16 = new_primitive_type("char16_t")
+    assert sizeof(BChar16) == 2
     _test_wchar_variant("char16_t")
+    assert int(cast(BChar16, -1)) == 0xffff       # always unsigned
 
 def test_char32():
-    assert sizeof(new_primitive_type("char32_t")) == 4
+    BChar32 = new_primitive_type("char32_t")
+    assert sizeof(BChar32) == 4
     _test_wchar_variant("char32_t")
+    assert int(cast(BChar32, -1)) == 0xffffffff   # always unsigned
 
 def _test_wchar_variant(typename):
     BWChar = new_primitive_type(typename)
diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -4,7 +4,6 @@
 
 typedef uint16_t cffi_char16_t;
 typedef uint32_t cffi_char32_t;
-/* NB. cffi_char32_t is unsigned to make the logic here a bit easier */
 
 
 #if Py_UNICODE_SIZE == 2
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -64,7 +64,7 @@
 # endif
 # if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
     typedef uint16_t char16_t;
-    typedef int32_t char32_t;
+    typedef uint32_t char32_t;
 # endif
 #else
 # include <stdint.h>

From pypy.commits at gmail.com  Fri Jun  2 04:11:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 01:11:26 -0700 (PDT)
Subject: [pypy-commit] cffi default: An extra test to distinguish wchar_t
 from char32_t, at least on linux
Message-ID: <59311dae.88a3190a.e3e30.bea2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2966:46cd2faf0897
Date: 2017-06-02 10:11 +0200
http://bitbucket.org/cffi/cffi/changeset/46cd2faf0897/

Log:	An extra test to distinguish wchar_t from char32_t, at least on
	linux

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2099,6 +2099,10 @@
 
 def test_wchar():
     _test_wchar_variant("wchar_t")
+    if sys.platform.startswith("linux"):
+        BWChar = new_primitive_type("wchar_t")
+        assert sizeof(BWChar) == 4
+        assert int(cast(BWChar, -1)) == -1        # signed, on linux
 
 def test_char16():
     BChar16 = new_primitive_type("char16_t")

From pypy.commits at gmail.com  Fri Jun  2 04:16:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 01:16:29 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document the signedness of charN_t
Message-ID: <59311edd.c54a190a.65120.1092@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2967:6c855193e59c
Date: 2017-06-02 10:16 +0200
http://bitbucket.org/cffi/cffi/changeset/6c855193e59c/

Log:	Document the signedness of charN_t

diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -629,8 +629,8 @@
 |               | or another <cdata char>| length 1         | ``<``          |
 +---------------+------------------------+------------------+----------------+
 | ``wchar_t``,  | a unicode of length 1  | a unicode of     |                |
-| ``char16_t``, | (or maybe 2 if         | length 1         | int(), bool(), |
-| ``char32_t``  | surrogates) or         | (or maybe 2 if   | ``<``          |
+| ``char16_t``, | (or maybe 2 if         | length 1         | int() `[8]`,   |
+| ``char32_t``  | surrogates) or         | (or maybe 2 if   | bool(), ``<``  |
 |               | another similar <cdata>| surrogates)      |                |
 +---------------+------------------------+------------------+----------------+
 |  ``float``,   | a float or anything on | a Python float   | float(), int(),|
@@ -774,6 +774,13 @@
    take directly as argument types or return type a complex type cannot
    be called by CFFI, unless they are directly using the API mode.
 
+`[8]` sign of ``wchar_t``, ``char16_t`` and ``char32_t``
+
+   The ``wchar_t`` type has the same signedness as the underlying
+   platform's.  For example, on Linux, it is a signed 32-bit integer.
+   However, the types ``char16_t`` and ``char32_t`` (*new in version
+   1.11*) are always unsigned.
+
 .. _file:
 
 Support for FILE

From pypy.commits at gmail.com  Fri Jun  2 05:45:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 02:45:18 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-complex: Reclose branch (oops)
Message-ID: <593133ae.91d81c0a.26dcd.e141@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-complex
Changeset: r91485:883157b06da5
Date: 2017-06-02 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/883157b06da5/

Log:	Reclose branch (oops)


From pypy.commits at gmail.com  Fri Jun  2 05:45:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 02:45:20 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: in-progress
Message-ID: <593133b0.08811c0a.67861.66f4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91486:e1a00d29e987
Date: 2017-06-02 11:36 +0200
http://bitbucket.org/pypy/pypy/changeset/e1a00d29e987/

Log:	in-progress

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.tool import rfficache
 
 from pypy.interpreter.error import oefmt
-from pypy.module._cffi_backend import cdataobj, misc
+from pypy.module._cffi_backend import cdataobj, misc, wchar_helper
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
@@ -148,50 +148,66 @@
         return self.space.newbytes(s)
 
 
-# XXX explicitly use an integer type instead of lltype.UniChar here,
-# because for now the latter is defined as unsigned by RPython (even
-# though it may be signed when 'wchar_t' is written to C).
-WCHAR_INT = {(2, False): rffi.USHORT,
-             (4, False): rffi.UINT,
-             (4, True): rffi.INT}[rffi.sizeof(lltype.UniChar),
-                                  rfficache.signof_c_type('wchar_t')]
-WCHAR_INTP = rffi.CArrayPtr(WCHAR_INT)
+class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
+    _attrs_            = ['is_signed']
+    _immutable_fields_ = ['is_signed']
 
-class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
-    _attrs_ = []
+    _wchar_is_signed = rfficache.signof_c_type('wchar_t')
 
-    if rffi.r_wchar_t.SIGN:
-        def write_raw_integer_data(self, w_cdata, value):
-            w_cdata.write_raw_signed_data(value)
+    def __init__(self, space, size, name, name_position, align):
+        W_CTypePrimitiveUniChar.__init__(self, space, size, name,
+                                         name_position, align)
+        self.is_signed = self._wchar_is_signed and (name == "wchar_t")
+        # "char16_t" and "char32_t" are always unsigned
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.newint(unichardata[0])
+        if self.is_signed_wchar:
+            value = misc.read_raw_long_data(cdata, self.size)
+            return self.space.newint(value)
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)
+            if self.size < rffi.sizeof(lltype.Signed):
+                return self.space.newint(intmask(value))
+            else:
+                return self.space.newint(value)    # r_uint => 'long' object
 
     def convert_to_object(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.newunicode(unichardata[0])
+        if self.is_signed_wchar:
+            unichardata = rffi.cast(rffi.CWCHARP, cdata)
+            return self.space.newunicode(unichardata[0])
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)   # r_uint
+            u = wchar_helper.ordinal_to_unicode(value)
+            return self.space.newunicode(u)
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
             w_res = self.convert_to_object(ptr)
         return w_res
 
-    def _convert_to_unichar(self, w_ob):
+    def _convert_to_charN_t(self, w_ob, size):
+        # returns a r_uint.  If size == 2, it is smaller than 0x10000
         space = self.space
         if space.isinstance_w(w_ob, space.w_unicode):
-            s = space.unicode_w(w_ob)
-            if len(s) == 1:
-                return s[0]
-        if (isinstance(w_ob, cdataobj.W_CData) and
-               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
+            u = space.unicode_w(w_ob)
+            if len(u) == 1:
+                u = ord(u[0])
+                if size == 2 and u > 0xffff:
+                    raise self._convert_error("single character <= 0xFFFF",
+                                              w_ob)
+                return r_uint(u)
+            elif size == 4 and len(u) == 2 and ...
+
+        elif (isinstance(w_ob, cdataobj.W_CData) and
+               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
+               w_ob.ctype.size == 2):
             with w_ob as ptr:
-                return rffi.cast(rffi.CWCHARP, ptr)[0]
+                return misc.read_raw_ulong_data(ptr, 2)
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
-        value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        ordinal = self._convert_to_char16(w_ob, self.size)
+        misc.write_raw_unsigned_data(cdata, ordinal, self.size)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
         u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -111,6 +111,9 @@
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+eptypesize("char16_t", 2, ctypeprim.W_CTypePrimitiveUniChar)
+eptypesize("char32_t", 4, ctypeprim.W_CTypePrimitiveUniChar)
+
 _WCTSigned = ctypeprim.W_CTypePrimitiveSigned
 _WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2087,22 +2087,40 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+
+def test_char16():
+    BChar16 = new_primitive_type("char16_t")
+    assert sizeof(BChar16) == 2
+    _test_wchar_variant("char16_t")
+    assert int(cast(BChar16, -1)) == 0xffff       # always unsigned
+
+def test_char32():
+    BChar32 = new_primitive_type("char32_t")
+    assert sizeof(BChar32) == 4
+    _test_wchar_variant("char32_t")
+    assert int(cast(BChar32, -1)) == 0xffffffff   # always unsigned
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2117,9 +2135,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2154,17 +2172,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2172,8 +2190,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2200,7 +2218,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -0,0 +1,17 @@
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+SIZE_UNICHAR = rffi.sizeof(lltype.UniChar)
+
+
+if SIZE_UNICODE == 4:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        return unichr(intmask(ordinal))
+else:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        if ordinal <= 0xffff:
+            return unichr(intmask(ordinal))
+        else:
+            ordinal = intmask(ordinal - 0x10000)
+            return (unichr(0xD800 | (ordinal >> 10)) +
+                    unichr(0xDC00 | (ordinal & 0x3FF)))

From pypy.commits at gmail.com  Fri Jun  2 05:55:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 02:55:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: __pypy__.StringBuilder: after build(),
 we can continue to append more
Message-ID: <593135fa.c7571c0a.e695b.d7f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91487:2dbc1ac2dab9
Date: 2017-06-02 11:54 +0200
http://bitbucket.org/pypy/pypy/changeset/2dbc1ac2dab9/

Log:	__pypy__.StringBuilder: after build(), we can continue to append
	more strings to the same builder. This is supported since
	2ff5087aca28 in RPython.

diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -18,31 +18,25 @@
             else:
                 self.builder = builder_cls(size)
 
-        def _check_done(self, space):
-            if self.builder is None:
-                raise oefmt(space.w_ValueError,
-                            "Can't operate on a built builder")
-
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
             return W_Builder(space, size)
 
         @unwrap_spec(s=unwrap)
         def descr_append(self, space, s):
-            self._check_done(space)
             self.builder.append(s)
 
         @unwrap_spec(s=unwrap, start=int, end=int)
         def descr_append_slice(self, space, s, start, end):
-            self._check_done(space)
             if not 0 <= start <= end <= len(s):
                 raise oefmt(space.w_ValueError, "bad start/stop")
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
-            self._check_done(space)
             w_s = getattr(space, newmethod)(self.builder.build())
-            self.builder = None
+            # after build(), we can continue to append more strings
+            # to the same builder.  This is supported since
+            # 2ff5087aca28 in RPython.
             return w_s
 
         def descr_len(self, space):
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -9,8 +9,9 @@
         b.append(u"1")
         s = b.build()
         assert s == u"abc1231"
-        raises(ValueError, b.build)
-        raises(ValueError, b.append, u"123")
+        assert b.build() == s
+        b.append(u"123")
+        assert b.build() == s + u"123"
 
     def test_preallocate(self):
         from __pypy__.builders import UnicodeBuilder
@@ -26,8 +27,9 @@
         b.append_slice(u"abcdefgh", 2, 5)
         raises(ValueError, b.append_slice, u"1", 2, 1)
         s = b.build()
-        assert s == "cde"
-        raises(ValueError, b.append_slice, u"abc", 1, 2)
+        assert s == u"cde"
+        b.append_slice(u"abc", 1, 2)
+        assert b.build() == u"cdeb"
 
     def test_stringbuilder(self):
         from __pypy__.builders import StringBuilder
@@ -37,6 +39,6 @@
         assert len(b) == 6
         b.append("you and me")
         s = b.build()
-        raises(ValueError, len, b)
+        assert len(b) == 16
         assert s == "abc123you and me"
-        raises(ValueError, b.build)
+        assert b.build() == s

From pypy.commits at gmail.com  Fri Jun  2 06:51:45 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:45 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: merge cffi 1.10.1 (from
 relevant parts of "hg -R cffi diff --rev 2919 --rev 2933")
Message-ID: <59314341.eb8edf0a.460a.9e5c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91489:e9ef3455dec5
Date: 2017-06-02 13:38 +0300
http://bitbucket.org/pypy/pypy/changeset/e9ef3455dec5/

Log:	merge cffi 1.10.1 (from relevant parts of "hg -R cffi diff --rev
	2919 --rev 2933")

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -1479,6 +1479,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:

From pypy.commits at gmail.com  Fri Jun  2 06:51:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:48 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge py3.5 into release
Message-ID: <59314344.918bdf0a.d7179.84f4@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91490:d5d9c4f1720c
Date: 2017-06-02 13:39 +0300
http://bitbucket.org/pypy/pypy/changeset/d5d9c4f1720c/

Log:	merge py3.5 into release

diff too long, truncating to 2000 out of 4287 lines

diff --git a/lib-python/2.7/ctypes/test/test_unaligned_structures.py b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
--- a/lib-python/2.7/ctypes/test/test_unaligned_structures.py
+++ b/lib-python/2.7/ctypes/test/test_unaligned_structures.py
@@ -37,7 +37,10 @@
         for typ in byteswapped_structures:
 ##            print >> sys.stderr, typ.value
             self.assertEqual(typ.value.offset, 1)
-            o = typ()
+            try:
+                o = typ()
+            except NotImplementedError as e:
+                self.skipTest(str(e))    # for PyPy
             o.value = 4
             self.assertEqual(o.value, 4)
 
diff --git a/lib-python/2.7/zipfile.py b/lib-python/2.7/zipfile.py
--- a/lib-python/2.7/zipfile.py
+++ b/lib-python/2.7/zipfile.py
@@ -622,19 +622,23 @@
         """Read and return up to n bytes.
         If the argument is omitted, None, or negative, data is read and returned until EOF is reached..
         """
-        buf = ''
+        # PyPy modification: don't do repeated string concatenation
+        buf = []
+        lenbuf = 0
         if n is None:
             n = -1
         while True:
             if n < 0:
                 data = self.read1(n)
-            elif n > len(buf):
-                data = self.read1(n - len(buf))
+            elif n > lenbuf:
+                data = self.read1(n - lenbuf)
             else:
-                return buf
+                break
             if len(data) == 0:
-                return buf
-            buf += data
+                break
+            lenbuf += len(data)
+            buf.append(data)
+        return "".join(buf)
 
     def _update_crc(self, newdata, eof):
         # Update the CRC using the given data.
diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py
--- a/lib-python/3/test/test_capi.py
+++ b/lib-python/3/test/test_capi.py
@@ -26,6 +26,11 @@
 # Were we compiled --with-pydebug or with #define Py_DEBUG?
 Py_DEBUG = hasattr(sys, 'gettotalrefcount')
 
+skips = []
+if support.check_impl_detail(pypy=True):
+    skips += [
+            'test_widechar',
+            ]
 
 def testfunction(self):
     """some doc"""
@@ -558,7 +563,7 @@
 class Test_testcapi(unittest.TestCase):
     def test__testcapi(self):
         for name in dir(_testcapi):
-            if name.startswith('test_'):
+            if name.startswith('test_') and name not in skips:
                 with self.subTest("internal", name=name):
                     test = getattr(_testcapi, name)
                     test()
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -1,5 +1,5 @@
-/* Verbatim copy of Modules/_testcapimodule.c from CPython 3.3,
- * except that "run_in_subinterp" is commented out
+/* Copy of Modules/_testcapimodule.c from CPython 3.3,
+ * with things that PyPy doesn't support disabled.
  */
 /*
  * C Extension module to test Python interpreter C APIs.
@@ -14,6 +14,12 @@
 #include <float.h>
 #include "structmember.h"
 #include "datetime.h"
+// #include "marshal.h"
+#include <signal.h>
+
+#ifdef MS_WINDOWS
+#  include <winsock2.h>         /* struct timeval */
+#endif
 
 #ifdef WITH_THREAD
 #include "pythread.h"
@@ -68,6 +74,76 @@
 }
 
 static PyObject*
+test_sizeof_c_types(PyObject *self)
+{
+#if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wtype-limits"
+#endif
+#define CHECK_SIZEOF(TYPE, EXPECTED)         \
+    if (EXPECTED != sizeof(TYPE))  {         \
+        PyErr_Format(TestError,              \
+            "sizeof(%s) = %u instead of %u", \
+            #TYPE, sizeof(TYPE), EXPECTED);  \
+        return (PyObject*)NULL;              \
+    }
+#define IS_SIGNED(TYPE) (((TYPE)-1) < (TYPE)0)
+#define CHECK_SIGNNESS(TYPE, SIGNED)         \
+    if (IS_SIGNED(TYPE) != SIGNED) {         \
+        PyErr_Format(TestError,              \
+            "%s signness is, instead of %i",  \
+            #TYPE, IS_SIGNED(TYPE), SIGNED); \
+        return (PyObject*)NULL;              \
+    }
+
+    /* integer types */
+    CHECK_SIZEOF(Py_UCS1, 1);
+    CHECK_SIZEOF(Py_UCS2, 2);
+    CHECK_SIZEOF(Py_UCS4, 4);
+    CHECK_SIGNNESS(Py_UCS1, 0);
+    CHECK_SIGNNESS(Py_UCS2, 0);
+    CHECK_SIGNNESS(Py_UCS4, 0);
+#ifdef HAVE_INT32_T
+    CHECK_SIZEOF(PY_INT32_T, 4);
+    CHECK_SIGNNESS(PY_INT32_T, 1);
+#endif
+#ifdef HAVE_UINT32_T
+    CHECK_SIZEOF(PY_UINT32_T, 4);
+    CHECK_SIGNNESS(PY_UINT32_T, 0);
+#endif
+#ifdef HAVE_INT64_T
+    CHECK_SIZEOF(PY_INT64_T, 8);
+    CHECK_SIGNNESS(PY_INT64_T, 1);
+#endif
+#ifdef HAVE_UINT64_T
+    CHECK_SIZEOF(PY_UINT64_T, 8);
+    CHECK_SIGNNESS(PY_UINT64_T, 0);
+#endif
+
+    /* pointer/size types */
+    CHECK_SIZEOF(size_t, sizeof(void *));
+    CHECK_SIGNNESS(size_t, 0);
+    CHECK_SIZEOF(Py_ssize_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_ssize_t, 1);
+
+    CHECK_SIZEOF(Py_uintptr_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_uintptr_t, 0);
+    CHECK_SIZEOF(Py_intptr_t, sizeof(void *));
+    CHECK_SIGNNESS(Py_intptr_t, 1);
+
+    Py_INCREF(Py_None);
+    return Py_None;
+
+#undef IS_SIGNED
+#undef CHECK_SIGNESS
+#undef CHECK_SIZEOF
+#if defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))
+#pragma GCC diagnostic pop
+#endif
+}
+
+
+static PyObject*
 test_list_api(PyObject *self)
 {
     PyObject* list;
@@ -175,6 +251,17 @@
     return Py_None;
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject*
+dict_hassplittable(PyObject *self, PyObject *arg)
+{
+    if (!PyArg_Parse(arg, "O!:dict_hassplittable", &PyDict_Type, &arg)) {
+        return NULL;
+    }
+    return PyBool_FromLong(_PyDict_HasSplitTable((PyDictObject*)arg));
+}
+#endif  /* PYPY_VERSION */
 
 /* Issue #4701: Check that PyObject_Hash implicitly calls
  *   PyType_Ready if it hasn't already been called
@@ -799,6 +886,120 @@
 
 #endif  /* ifdef HAVE_LONG_LONG */
 
+static PyObject *
+return_none(void *unused)
+{
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+raise_error(void *unused)
+{
+    PyErr_SetNone(PyExc_ValueError);
+    return NULL;
+}
+
+static int
+test_buildvalue_N_error(const char *fmt)
+{
+    PyObject *arg, *res;
+
+    arg = PyList_New(0);
+    if (arg == NULL) {
+        return -1;
+    }
+
+    Py_INCREF(arg);
+    res = Py_BuildValue(fmt, return_none, NULL, arg);
+    if (res == NULL) {
+        return -1;
+    }
+    Py_DECREF(res);
+    if (Py_REFCNT(arg) != 1) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "arg was not decrefed in successful "
+                     "Py_BuildValue(\"%s\")", fmt);
+        return -1;
+    }
+
+    Py_INCREF(arg);
+    res = Py_BuildValue(fmt, raise_error, NULL, arg);
+    if (res != NULL || !PyErr_Occurred()) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "Py_BuildValue(\"%s\") didn't complain", fmt);
+        return -1;
+    }
+    PyErr_Clear();
+    if (Py_REFCNT(arg) != 1) {
+        PyErr_Format(TestError, "test_buildvalue_N: "
+                     "arg was not decrefed in failed "
+                     "Py_BuildValue(\"%s\")", fmt);
+        return -1;
+    }
+    Py_DECREF(arg);
+    return 0;
+}
+
+static PyObject *
+test_buildvalue_N(PyObject *self, PyObject *noargs)
+{
+    PyObject *arg, *res;
+
+    arg = PyList_New(0);
+    if (arg == NULL) {
+        return NULL;
+    }
+    Py_INCREF(arg);
+    res = Py_BuildValue("N", arg);
+    if (res == NULL) {
+        return NULL;
+    }
+    if (res != arg) {
+        return raiseTestError("test_buildvalue_N",
+                              "Py_BuildValue(\"N\") returned wrong result");
+    }
+    if (Py_REFCNT(arg) != 2) {
+        return raiseTestError("test_buildvalue_N",
+                              "arg was not decrefed in Py_BuildValue(\"N\")");
+    }
+    Py_DECREF(res);
+    Py_DECREF(arg);
+
+    if (test_buildvalue_N_error("O&N") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("(O&N)") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("[O&N]") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("{O&N}") < 0)
+        return NULL;
+    if (test_buildvalue_N_error("{()O&(())N}") < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
+
+static PyObject *
+get_args(PyObject *self, PyObject *args)
+{
+    if (args == NULL) {
+        args = Py_None;
+    }
+    Py_INCREF(args);
+    return args;
+}
+
+static PyObject *
+get_kwargs(PyObject *self, PyObject *args, PyObject *kwargs)
+{
+    if (kwargs == NULL) {
+        kwargs = Py_None;
+    }
+    Py_INCREF(kwargs);
+    return kwargs;
+}
+
 /* Test tuple argument processing */
 static PyObject *
 getargs_tuple(PyObject *self, PyObject *args)
@@ -1010,12 +1211,78 @@
 }
 
 static PyObject *
+getargs_f(PyObject *self, PyObject *args)
+{
+    float f;
+    if (!PyArg_ParseTuple(args, "f", &f))
+        return NULL;
+    return PyFloat_FromDouble(f);
+}
+
+static PyObject *
+getargs_d(PyObject *self, PyObject *args)
+{
+    double d;
+    if (!PyArg_ParseTuple(args, "d", &d))
+        return NULL;
+    return PyFloat_FromDouble(d);
+}
+
+static PyObject *
+getargs_D(PyObject *self, PyObject *args)
+{
+    Py_complex cval;
+    if (!PyArg_ParseTuple(args, "D", &cval))
+        return NULL;
+    return PyComplex_FromCComplex(cval);
+}
+
+static PyObject *
+getargs_S(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "S", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
+getargs_Y(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "Y", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
+getargs_U(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    if (!PyArg_ParseTuple(args, "U", &obj))
+        return NULL;
+    Py_INCREF(obj);
+    return obj;
+}
+
+static PyObject *
 getargs_c(PyObject *self, PyObject *args)
 {
     char c;
     if (!PyArg_ParseTuple(args, "c", &c))
         return NULL;
-    return PyBytes_FromStringAndSize(&c, 1);
+    return PyLong_FromLong((unsigned char)c);
+}
+
+static PyObject *
+getargs_C(PyObject *self, PyObject *args)
+{
+    int c;
+    if (!PyArg_ParseTuple(args, "C", &c))
+        return NULL;
+    return PyLong_FromLong(c);
 }
 
 static PyObject *
@@ -1170,6 +1437,84 @@
         Py_RETURN_NONE;
 }
 
+static PyObject *
+getargs_es(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    char *str;
+
+    if (!PyArg_ParseTuple(args, "O|s", &arg, &encoding))
+        return NULL;
+    if (!PyArg_Parse(arg, "es", encoding, &str))
+        return NULL;
+    result = PyBytes_FromString(str);
+    PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_et(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    char *str;
+
+    if (!PyArg_ParseTuple(args, "O|s", &arg, &encoding))
+        return NULL;
+    if (!PyArg_Parse(arg, "et", encoding, &str))
+        return NULL;
+    result = PyBytes_FromString(str);
+    PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_es_hash(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    PyByteArrayObject *buffer = NULL;
+    char *str = NULL;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "O|sY", &arg, &encoding, &buffer))
+        return NULL;
+    if (buffer != NULL) {
+        str = PyByteArray_AS_STRING(buffer);
+        size = PyByteArray_GET_SIZE(buffer);
+    }
+    if (!PyArg_Parse(arg, "es#", encoding, &str, &size))
+        return NULL;
+    result = PyBytes_FromStringAndSize(str, size);
+    if (buffer == NULL)
+        PyMem_Free(str);
+    return result;
+}
+
+static PyObject *
+getargs_et_hash(PyObject *self, PyObject *args)
+{
+    PyObject *arg, *result;
+    const char *encoding = NULL;
+    PyByteArrayObject *buffer = NULL;
+    char *str = NULL;
+    Py_ssize_t size;
+
+    if (!PyArg_ParseTuple(args, "O|sY", &arg, &encoding, &buffer))
+        return NULL;
+    if (buffer != NULL) {
+        str = PyByteArray_AS_STRING(buffer);
+        size = PyByteArray_GET_SIZE(buffer);
+    }
+    if (!PyArg_Parse(arg, "et#", encoding, &str, &size))
+        return NULL;
+    result = PyBytes_FromStringAndSize(str, size);
+    if (buffer == NULL)
+        PyMem_Free(str);
+    return result;
+}
+
 /* Test the s and z codes for PyArg_ParseTuple.
 */
 static PyObject *
@@ -1427,6 +1772,20 @@
     else
         return raiseTestError("test_widechar",
                               "PyUnicode_FromUnicode(L\"\\U00110000\", 1) didn't fail");
+
+    wide = PyUnicode_FromUnicode(NULL, 1);
+    if (wide == NULL)
+        return NULL;
+    PyUnicode_AS_UNICODE(wide)[0] = invalid[0];
+    if (_PyUnicode_Ready(wide) < 0) {
+        Py_DECREF(wide);
+        PyErr_Clear();
+    }
+    else {
+        Py_DECREF(wide);
+        return raiseTestError("test_widechar",
+                              "PyUnicode_Ready() didn't fail");
+    }
 #endif
 
     Py_RETURN_NONE;
@@ -1441,7 +1800,7 @@
 
     if (!PyArg_ParseTuple(args, "Un", &unicode, &buflen))
         return NULL;
-    buffer = PyMem_Malloc(buflen * sizeof(wchar_t));
+    buffer = PyMem_New(wchar_t, buflen);
     if (buffer == NULL)
         return PyErr_NoMemory();
 
@@ -1484,6 +1843,72 @@
     return Py_BuildValue("(Nn)", result, size);
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject *
+unicode_asucs4(PyObject *self, PyObject *args)
+{
+    PyObject *unicode, *result;
+    Py_UCS4 *buffer;
+    int copy_null;
+    Py_ssize_t str_len, buf_len;
+
+    if (!PyArg_ParseTuple(args, "Unp:unicode_asucs4", &unicode, &str_len, &copy_null)) {
+        return NULL;
+    }
+
+    buf_len = str_len + 1;
+    buffer = PyMem_NEW(Py_UCS4, buf_len);
+    if (buffer == NULL) {
+        return PyErr_NoMemory();
+    }
+    memset(buffer, 0, sizeof(Py_UCS4)*buf_len);
+    buffer[str_len] = 0xffffU;
+
+    if (!PyUnicode_AsUCS4(unicode, buffer, buf_len, copy_null)) {
+        PyMem_FREE(buffer);
+        return NULL;
+    }
+
+    result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buffer, buf_len);
+    PyMem_FREE(buffer);
+    return result;
+}
+
+static PyObject *
+unicode_copycharacters(PyObject *self, PyObject *args)
+{
+    PyObject *from, *to, *to_copy;
+    Py_ssize_t from_start, to_start, how_many, copied;
+
+    if (!PyArg_ParseTuple(args, "UnOnn:unicode_copycharacters", &to, &to_start,
+                          &from, &from_start, &how_many)) {
+        return NULL;
+    }
+
+    if (PyUnicode_READY(to) < 0) {
+        return NULL;
+    }
+
+    if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to),
+                                  PyUnicode_MAX_CHAR_VALUE(to)))) {
+        return NULL;
+    }
+    if (PyUnicode_Fill(to_copy, 0, PyUnicode_GET_LENGTH(to_copy), 0U) < 0) {
+        Py_DECREF(to_copy);
+        return NULL;
+    }
+
+    if ((copied = PyUnicode_CopyCharacters(to_copy, to_start, from,
+                                           from_start, how_many)) < 0) {
+        Py_DECREF(to_copy);
+        return NULL;
+    }
+
+    return Py_BuildValue("(Nn)", to_copy, copied);
+}
+#endif  /* PYPY_VERSION */
+
 static PyObject *
 unicode_encodedecimal(PyObject *self, PyObject *args)
 {
@@ -1644,7 +2069,7 @@
              {-0xffffL, 16, -1},
              {0xfffffffL, 28, 1},
              {-0xfffffffL, 28, -1}};
-    int i;
+    size_t i;
 
     for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) {
         size_t nbits;
@@ -1709,6 +2134,18 @@
 }
 
 static PyObject *
+set_errno(PyObject *self, PyObject *args)
+{
+    int new_errno;
+
+    if (!PyArg_ParseTuple(args, "i:set_errno", &new_errno))
+        return NULL;
+
+    errno = new_errno;
+    Py_RETURN_NONE;
+}
+
+static PyObject *
 test_set_exc_info(PyObject *self, PyObject *args)
 {
     PyObject *orig_exc;
@@ -1879,6 +2316,8 @@
 }
 #endif
 
+#ifndef PYPY_VERSION
+
 /* Some tests of PyUnicode_FromFormat().  This needs more tests. */
 static PyObject *
 test_string_from_format(PyObject *self, PyObject *args)
@@ -1890,7 +2329,7 @@
     result = PyUnicode_FromFormat(FORMAT, (TYPE)1);                 \
     if (result == NULL)                                             \
         return NULL;                                                \
-    if (PyUnicode_CompareWithASCIIString(result, "1")) {     \
+    if (!_PyUnicode_EqualToASCIIString(result, "1")) {              \
         msg = FORMAT " failed at 1";                                \
         goto Fail;                                                  \
     }                                                               \
@@ -1920,6 +2359,7 @@
 
 #undef CHECK_1_FORMAT
 }
+#endif  /* PYPY_VERSION */
 
 
 static PyObject *
@@ -2394,6 +2834,111 @@
     return PyMemoryView_FromBuffer(&info);
 }
 
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_from_contiguous(PyObject* self, PyObject *noargs)
+{
+    int data[9] = {-1,-1,-1,-1,-1,-1,-1,-1,-1};
+    int init[5] = {0, 1, 2, 3, 4};
+    Py_ssize_t itemsize = sizeof(int);
+    Py_ssize_t shape = 5;
+    Py_ssize_t strides = 2 * itemsize;
+    Py_buffer view = {
+        data,
+        NULL,
+        5 * itemsize,
+        itemsize,
+        1,
+        1,
+        NULL,
+        &shape,
+        &strides,
+        NULL,
+        NULL
+    };
+    int *ptr;
+    int i;
+
+    PyBuffer_FromContiguous(&view, init, view.len, 'C');
+    ptr = view.buf;
+    for (i = 0; i < 5; i++) {
+        if (ptr[2*i] != i) {
+            PyErr_SetString(TestError,
+                "test_from_contiguous: incorrect result");
+            return NULL;
+        }
+    }
+
+    view.buf = &data[8];
+    view.strides[0] = -2 * itemsize;
+
+    PyBuffer_FromContiguous(&view, init, view.len, 'C');
+    ptr = view.buf;
+    for (i = 0; i < 5; i++) {
+        if (*(ptr-2*i) != i) {
+            PyErr_SetString(TestError,
+                "test_from_contiguous: incorrect result");
+            return NULL;
+        }
+    }
+
+    Py_RETURN_NONE;
+}
+#endif  /* PYPY_VERSION */
+
+#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION)
+extern PyTypeObject _PyBytesIOBuffer_Type;
+
+static PyObject *
+test_pep3118_obsolete_write_locks(PyObject* self, PyObject *noargs)
+{
+    PyTypeObject *type = &_PyBytesIOBuffer_Type;
+    PyObject *b;
+    char *dummy[1];
+    int ret, match;
+
+    /* PyBuffer_FillInfo() */
+    ret = PyBuffer_FillInfo(NULL, NULL, dummy, 1, 0, PyBUF_SIMPLE);
+    match = PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_BufferError);
+    PyErr_Clear();
+    if (ret != -1 || match == 0)
+        goto error;
+
+    /* bytesiobuf_getbuffer() */
+    b = type->tp_alloc(type, 0);
+    if (b == NULL) {
+        return NULL;
+    }
+
+    ret = PyObject_GetBuffer(b, NULL, PyBUF_SIMPLE);
+    Py_DECREF(b);
+    match = PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_BufferError);
+    PyErr_Clear();
+    if (ret != -1 || match == 0)
+        goto error;
+
+    Py_RETURN_NONE;
+
+error:
+    PyErr_SetString(TestError,
+        "test_pep3118_obsolete_write_locks: failure");
+    return NULL;
+}
+#endif
+
+/* This tests functions that historically supported write locks.  It is
+   wrong to call getbuffer() with view==NULL and a compliant getbufferproc
+   is entitled to segfault in that case. */
+static PyObject *
+getbuffer_with_null_view(PyObject* self, PyObject *obj)
+{
+    if (PyObject_GetBuffer(obj, NULL, PyBUF_SIMPLE) < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
 /* Test that the fatal error from not having a current thread doesn't
    cause an infinite loop.  Run via Lib/test/test_capi.py */
 static PyObject *
@@ -2444,14 +2989,27 @@
     return PyLong_FromLong(r);
 }
 
+static int
+check_time_rounding(int round)
+{
+    if (round != _PyTime_ROUND_FLOOR && round != _PyTime_ROUND_CEILING) {
+        PyErr_SetString(PyExc_ValueError, "invalid rounding");
+        return -1;
+    }
+    return 0;
+}
+
 static PyObject *
 test_pytime_object_to_time_t(PyObject *self, PyObject *args)
 {
     PyObject *obj;
     time_t sec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_time_t", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTime_t(obj, &sec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_time_t", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTime_t(obj, &sec, round) == -1)
         return NULL;
     return _PyLong_FromTime_t(sec);
 }
@@ -2462,9 +3020,12 @@
     PyObject *obj;
     time_t sec;
     long usec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_timeval", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTimeval(obj, &sec, &usec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timeval", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTimeval(obj, &sec, &usec, round) == -1)
         return NULL;
     return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), usec);
 }
@@ -2475,15 +3036,444 @@
     PyObject *obj;
     time_t sec;
     long nsec;
-    if (!PyArg_ParseTuple(args, "O:pytime_object_to_timespec", &obj))
-        return NULL;
-    if (_PyTime_ObjectToTimespec(obj, &sec, &nsec) == -1)
+    int round;
+    if (!PyArg_ParseTuple(args, "Oi:pytime_object_to_timespec", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_ObjectToTimespec(obj, &sec, &nsec, round) == -1)
         return NULL;
     return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec);
 }
 
+static void
+slot_tp_del(PyObject *self)
+{
+    _Py_IDENTIFIER(__tp_del__);
+    PyObject *del, *res;
+    PyObject *error_type, *error_value, *error_traceback;
+
+    /* Temporarily resurrect the object. */
+    assert(self->ob_refcnt == 0);
+    self->ob_refcnt = 1;
+
+    /* Save the current exception, if any. */
+    PyErr_Fetch(&error_type, &error_value, &error_traceback);
+
+    /* Execute __del__ method, if any. */
+    del = _PyObject_LookupSpecial(self, &PyId___tp_del__);
+    if (del != NULL) {
+        res = PyEval_CallObject(del, NULL);
+        if (res == NULL)
+            PyErr_WriteUnraisable(del);
+        else
+            Py_DECREF(res);
+        Py_DECREF(del);
+    }
+
+    /* Restore the saved exception. */
+    PyErr_Restore(error_type, error_value, error_traceback);
+
+    /* Undo the temporary resurrection; can't use DECREF here, it would
+     * cause a recursive call.
+     */
+    assert(self->ob_refcnt > 0);
+    if (--self->ob_refcnt == 0)
+        return;         /* this is the normal path out */
+
+    /* __del__ resurrected it!  Make it look like the original Py_DECREF
+     * never happened.
+     */
+    {
+        Py_ssize_t refcnt = self->ob_refcnt;
+        _Py_NewReference(self);
+        self->ob_refcnt = refcnt;
+    }
+    assert(!PyType_IS_GC(Py_TYPE(self)) ||
+           _Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
+    /* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
+     * we need to undo that. */
+    _Py_DEC_REFTOTAL;
+    /* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
+     * chain, so no more to do there.
+     * If COUNT_ALLOCS, the original decref bumped tp_frees, and
+     * _Py_NewReference bumped tp_allocs:  both of those need to be
+     * undone.
+     */
+#ifdef COUNT_ALLOCS
+    --Py_TYPE(self)->tp_frees;
+    --Py_TYPE(self)->tp_allocs;
+#endif
+}
+
+static PyObject *
+with_tp_del(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    PyTypeObject *tp;
+
+    if (!PyArg_ParseTuple(args, "O:with_tp_del", &obj))
+        return NULL;
+    tp = (PyTypeObject *) obj;
+    if (!PyType_Check(obj) || !PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE)) {
+        PyErr_Format(PyExc_TypeError,
+                     "heap type expected, got %R", obj);
+        return NULL;
+    }
+    tp->tp_del = slot_tp_del;
+    Py_INCREF(obj);
+    return obj;
+}
+
 #endif  /* PYPY_VERSION */
 
+static PyMethodDef ml;
+
+static PyObject *
+create_cfunction(PyObject *self, PyObject *args)
+{
+    return PyCFunction_NewEx(&ml, self, NULL);
+}
+
+static PyMethodDef ml = {
+    "create_cfunction",
+    create_cfunction,
+    METH_NOARGS,
+    NULL
+};
+
+static PyObject *
+_test_incref(PyObject *ob)
+{
+    Py_INCREF(ob);
+    return ob;
+}
+
+static PyObject *
+test_xincref_doesnt_leak(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_XINCREF(_test_incref(obj));
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_incref_doesnt_leak(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_INCREF(_test_incref(obj));
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_DECREF(obj);
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_xdecref_doesnt_leak(PyObject *ob)
+{
+    Py_XDECREF(PyLong_FromLong(0));
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_decref_doesnt_leak(PyObject *ob)
+{
+    Py_DECREF(PyLong_FromLong(0));
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+test_incref_decref_API(PyObject *ob)
+{
+    PyObject *obj = PyLong_FromLong(0);
+    Py_IncRef(obj);
+    Py_DecRef(obj);
+    Py_DecRef(obj);
+    Py_RETURN_NONE;
+}
+
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_pymem_alloc0(PyObject *self)
+{
+    void *ptr;
+
+    ptr = PyMem_RawMalloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawMalloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
+    ptr = PyMem_RawCalloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_RawCalloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_RawFree(ptr);
+
+    ptr = PyMem_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyMem_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyMem_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyMem_Free(ptr);
+
+    ptr = PyObject_Malloc(0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Malloc(0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    ptr = PyObject_Calloc(0, 0);
+    if (ptr == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "PyObject_Calloc(0, 0) returns NULL");
+        return NULL;
+    }
+    PyObject_Free(ptr);
+
+    Py_RETURN_NONE;
+}
+
+typedef struct {
+    PyMemAllocatorEx alloc;
+
+    size_t malloc_size;
+    size_t calloc_nelem;
+    size_t calloc_elsize;
+    void *realloc_ptr;
+    size_t realloc_new_size;
+    void *free_ptr;
+} alloc_hook_t;
+
+static void* hook_malloc (void* ctx, size_t size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->malloc_size = size;
+    return hook->alloc.malloc(hook->alloc.ctx, size);
+}
+
+static void* hook_calloc (void* ctx, size_t nelem, size_t elsize)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->calloc_nelem = nelem;
+    hook->calloc_elsize = elsize;
+    return hook->alloc.calloc(hook->alloc.ctx, nelem, elsize);
+}
+
+static void* hook_realloc (void* ctx, void* ptr, size_t new_size)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->realloc_ptr = ptr;
+    hook->realloc_new_size = new_size;
+    return hook->alloc.realloc(hook->alloc.ctx, ptr, new_size);
+}
+
+static void hook_free (void *ctx, void *ptr)
+{
+    alloc_hook_t *hook = (alloc_hook_t *)ctx;
+    hook->free_ptr = ptr;
+    hook->alloc.free(hook->alloc.ctx, ptr);
+}
+
+static PyObject *
+test_setallocators(PyMemAllocatorDomain domain)
+{
+    PyObject *res = NULL;
+    const char *error_msg;
+    alloc_hook_t hook;
+    PyMemAllocatorEx alloc;
+    size_t size, size2, nelem, elsize;
+    void *ptr, *ptr2;
+
+    memset(&hook, 0, sizeof(hook));
+
+    alloc.ctx = &hook;
+    alloc.malloc = &hook_malloc;
+    alloc.calloc = &hook_calloc;
+    alloc.realloc = &hook_realloc;
+    alloc.free = &hook_free;
+    PyMem_GetAllocator(domain, &hook.alloc);
+    PyMem_SetAllocator(domain, &alloc);
+
+    size = 42;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawMalloc(size); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Malloc(size); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Malloc(size); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "malloc failed";
+        goto fail;
+    }
+
+    if (hook.malloc_size != size) {
+        error_msg = "malloc invalid size";
+        goto fail;
+    }
+
+    size2 = 200;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr2 = PyMem_RawRealloc(ptr, size2); break;
+    case PYMEM_DOMAIN_MEM: ptr2 = PyMem_Realloc(ptr, size2); break;
+    case PYMEM_DOMAIN_OBJ: ptr2 = PyObject_Realloc(ptr, size2); break;
+    default: ptr2 = NULL; break;
+    }
+
+    if (ptr2 == NULL) {
+        error_msg = "realloc failed";
+        goto fail;
+    }
+
+    if (hook.realloc_ptr != ptr
+        || hook.realloc_new_size != size2) {
+        error_msg = "realloc invalid parameters";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr2); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr2); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr2); break;
+    }
+
+    if (hook.free_ptr != ptr2) {
+        error_msg = "free invalid pointer";
+        goto fail;
+    }
+
+    nelem = 2;
+    elsize = 5;
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: ptr = PyMem_RawCalloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_MEM: ptr = PyMem_Calloc(nelem, elsize); break;
+    case PYMEM_DOMAIN_OBJ: ptr = PyObject_Calloc(nelem, elsize); break;
+    default: ptr = NULL; break;
+    }
+
+    if (ptr == NULL) {
+        error_msg = "calloc failed";
+        goto fail;
+    }
+
+    if (hook.calloc_nelem != nelem || hook.calloc_elsize != elsize) {
+        error_msg = "calloc invalid nelem or elsize";
+        goto fail;
+    }
+
+    switch(domain)
+    {
+    case PYMEM_DOMAIN_RAW: PyMem_RawFree(ptr); break;
+    case PYMEM_DOMAIN_MEM: PyMem_Free(ptr); break;
+    case PYMEM_DOMAIN_OBJ: PyObject_Free(ptr); break;
+    }
+
+    Py_INCREF(Py_None);
+    res = Py_None;
+    goto finally;
+
+fail:
+    PyErr_SetString(PyExc_RuntimeError, error_msg);
+
+finally:
+    PyMem_SetAllocator(domain, &hook.alloc);
+    return res;
+}
+
+static PyObject *
+test_pymem_setrawallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_RAW);
+}
+
+static PyObject *
+test_pymem_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_MEM);
+}
+
+static PyObject *
+test_pyobject_setallocators(PyObject *self)
+{
+    return test_setallocators(PYMEM_DOMAIN_OBJ);
+}
+
+#endif  /* PYPY_VERSION */
+
+PyDoc_STRVAR(docstring_empty,
+""
+);
+
+PyDoc_STRVAR(docstring_no_signature,
+"This docstring has no signature."
+);
+
+PyDoc_STRVAR(docstring_with_invalid_signature,
+"docstring_with_invalid_signature($module, /, boo)\n"
+"\n"
+"This docstring has an invalid signature."
+);
+
+PyDoc_STRVAR(docstring_with_invalid_signature2,
+"docstring_with_invalid_signature2($module, /, boo)\n"
+"\n"
+"--\n"
+"\n"
+"This docstring also has an invalid signature."
+);
+
+PyDoc_STRVAR(docstring_with_signature,
+"docstring_with_signature($module, /, sig)\n"
+"--\n"
+"\n"
+"This docstring has a valid signature."
+);
+
+PyDoc_STRVAR(docstring_with_signature_but_no_doc,
+"docstring_with_signature_but_no_doc($module, /, sig)\n"
+"--\n"
+"\n"
+);
+
+PyDoc_STRVAR(docstring_with_signature_and_extra_newlines,
+"docstring_with_signature_and_extra_newlines($module, /, parameter)\n"
+"--\n"
+"\n"
+"\n"
+"This docstring has a valid signature and some extra newlines."
+);
+
+PyDoc_STRVAR(docstring_with_signature_with_defaults,
+"docstring_with_signature_with_defaults(module, s='avocado',\n"
+"        b=b'bytes', d=3.14, i=35, n=None, t=True, f=False,\n"
+"        local=the_number_three, sys=sys.maxsize,\n"
+"        exp=sys.maxsize - 1)\n"
+"--\n"
+"\n"
+"\n"
+"\n"
+"This docstring has a valid signature with parameters,\n"
+"and the parameters take defaults of varying types."
+);
+
 #ifdef WITH_THREAD
 typedef struct {
     PyThread_type_lock start_event;
@@ -2573,16 +3563,349 @@
 }
 #endif   /* WITH_THREAD */
 
+static PyObject*
+test_raise_signal(PyObject* self, PyObject *args)
+{
+    int signum, err;
+
+    if (PyArg_ParseTuple(args, "i:raise_signal", &signum) < 0)
+        return NULL;
+
+    err = raise(signum);
+    if (err)
+        return PyErr_SetFromErrno(PyExc_OSError);
+
+    if (PyErr_CheckSignals() < 0)
+        return NULL;
+
+    Py_RETURN_NONE;
+}
+
+/* marshal */
+
+#ifndef PYPY_VERSION
+
+static PyObject*
+pymarshal_write_long_to_file(PyObject* self, PyObject *args)
+{
+    long value;
+    char *filename;
+    int version;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "lsi:pymarshal_write_long_to_file",
+                          &value, &filename, &version))
+        return NULL;
+
+    fp = fopen(filename, "wb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    PyMarshal_WriteLongToFile(value, fp, version);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+pymarshal_write_object_to_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    char *filename;
+    int version;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "Osi:pymarshal_write_object_to_file",
+                          &obj, &filename, &version))
+        return NULL;
+
+    fp = fopen(filename, "wb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    PyMarshal_WriteObjectToFile(obj, fp, version);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    Py_RETURN_NONE;
+}
+
+static PyObject*
+pymarshal_read_short_from_file(PyObject* self, PyObject *args)
+{
+    int value;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_short_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    value = PyMarshal_ReadShortFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    return Py_BuildValue("il", value, pos);
+}
+
+static PyObject*
+pymarshal_read_long_from_file(PyObject* self, PyObject *args)
+{
+    long value, pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_long_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    value = PyMarshal_ReadLongFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    if (PyErr_Occurred())
+        return NULL;
+    return Py_BuildValue("ll", value, pos);
+}
+
+static PyObject*
+pymarshal_read_last_object_from_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_last_object_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    obj = PyMarshal_ReadLastObjectFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    return Py_BuildValue("Nl", obj, pos);
+}
+
+static PyObject*
+pymarshal_read_object_from_file(PyObject* self, PyObject *args)
+{
+    PyObject *obj;
+    long pos;
+    char *filename;
+    FILE *fp;
+
+    if (!PyArg_ParseTuple(args, "s:pymarshal_read_object_from_file", &filename))
+        return NULL;
+
+    fp = fopen(filename, "rb");
+    if (fp == NULL) {
+        PyErr_SetFromErrno(PyExc_OSError);
+        return NULL;
+    }
+
+    obj = PyMarshal_ReadObjectFromFile(fp);
+    pos = ftell(fp);
+
+    fclose(fp);
+    return Py_BuildValue("Nl", obj, pos);
+}
+#endif  /* PYPY_VERSION */
+
+static PyObject*
+return_null_without_error(PyObject *self, PyObject *args)
+{
+    /* invalid call: return NULL without setting an error,
+     * _Py_CheckFunctionResult() must detect such bug at runtime. */
+    PyErr_Clear();
+    return NULL;
+}
+
+static PyObject*
+return_result_with_error(PyObject *self, PyObject *args)
+{
+    /* invalid call: return a result with an error set,
+     * _Py_CheckFunctionResult() must detect such bug at runtime. */
+    PyErr_SetNone(PyExc_ValueError);
+    Py_RETURN_NONE;
+}
+
+#ifndef PYPY_VERSION
+
+static PyObject *
+test_pytime_fromseconds(PyObject *self, PyObject *args)
+{
+    int seconds;
+    _PyTime_t ts;
+
+    if (!PyArg_ParseTuple(args, "i", &seconds))
+        return NULL;
+    ts = _PyTime_FromSeconds(seconds);
+    return _PyTime_AsNanosecondsObject(ts);
+}
+
+static PyObject *
+test_pytime_fromsecondsobject(PyObject *self, PyObject *args)
+{
+    PyObject *obj;
+    int round;
+    _PyTime_t ts;
+
+    if (!PyArg_ParseTuple(args, "Oi", &obj, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    if (_PyTime_FromSecondsObject(&ts, obj, round) == -1)
+        return NULL;
+    return _PyTime_AsNanosecondsObject(ts);
+}
+
+static PyObject *
+test_pytime_assecondsdouble(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    _PyTime_t ts;
+    double d;
+
+    if (!PyArg_ParseTuple(args, "L", &ns))
+        return NULL;
+    ts = _PyTime_FromNanoseconds(ns);
+    d = _PyTime_AsSecondsDouble(ts);
+    return PyFloat_FromDouble(d);
+}
+
+static PyObject *
+test_PyTime_AsTimeval(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t;
+    struct timeval tv;
+    PyObject *seconds;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    if (_PyTime_AsTimeval(t, &tv, round) < 0)
+        return NULL;
+
+    seconds = PyLong_FromLong((PY_LONG_LONG)tv.tv_sec);
+    if (seconds == NULL)
+        return NULL;
+    return Py_BuildValue("Nl", seconds, tv.tv_usec);
+}
+
+#ifdef HAVE_CLOCK_GETTIME
+static PyObject *
+test_PyTime_AsTimespec(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    _PyTime_t t;
+    struct timespec ts;
+
+    if (!PyArg_ParseTuple(args, "L", &ns))
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    if (_PyTime_AsTimespec(t, &ts) == -1)
+        return NULL;
+    return Py_BuildValue("Nl", _PyLong_FromTime_t(ts.tv_sec), ts.tv_nsec);
+}
+#endif
+
+static PyObject *
+test_PyTime_AsMilliseconds(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t, ms;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    ms = _PyTime_AsMilliseconds(t, round);
+    /* This conversion rely on the fact that _PyTime_t is a number of
+       nanoseconds */
+    return _PyTime_AsNanosecondsObject(ms);
+}
+
+static PyObject *
+test_PyTime_AsMicroseconds(PyObject *self, PyObject *args)
+{
+    PY_LONG_LONG ns;
+    int round;
+    _PyTime_t t, ms;
+
+    if (!PyArg_ParseTuple(args, "Li", &ns, &round))
+        return NULL;
+    if (check_time_rounding(round) < 0)
+        return NULL;
+    t = _PyTime_FromNanoseconds(ns);
+    ms = _PyTime_AsMicroseconds(t, round);
+    /* This conversion rely on the fact that _PyTime_t is a number of
+       nanoseconds */
+    return _PyTime_AsNanosecondsObject(ms);
+}
+
+
+static PyObject*
+get_recursion_depth(PyObject *self, PyObject *args)
+{
+    PyThreadState *tstate = PyThreadState_GET();
+
+    /* subtract one to ignore the frame of the get_recursion_depth() call */
+    return PyLong_FromLong(tstate->recursion_depth - 1);
+}
+
+#endif  /* PYPY_VERSION */
 
 static PyMethodDef TestMethods[] = {
     {"raise_exception",         raise_exception,                 METH_VARARGS},
     {"raise_memoryerror",   (PyCFunction)raise_memoryerror,  METH_NOARGS},
+    {"set_errno",               set_errno,                       METH_VARARGS},
     {"test_config",             (PyCFunction)test_config,        METH_NOARGS},
+    {"test_sizeof_c_types",     (PyCFunction)test_sizeof_c_types, METH_NOARGS},
     {"test_datetime_capi",  test_datetime_capi,              METH_NOARGS},
     {"test_list_api",           (PyCFunction)test_list_api,      METH_NOARGS},
     {"test_dict_iteration",     (PyCFunction)test_dict_iteration,METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"dict_hassplittable",      dict_hassplittable,              METH_O},
+#endif
     {"test_lazy_hash_inheritance",      (PyCFunction)test_lazy_hash_inheritance,METH_NOARGS},
     {"test_long_api",           (PyCFunction)test_long_api,      METH_NOARGS},
+    {"test_xincref_doesnt_leak",(PyCFunction)test_xincref_doesnt_leak,      METH_NOARGS},
+    {"test_incref_doesnt_leak", (PyCFunction)test_incref_doesnt_leak,      METH_NOARGS},
+    {"test_xdecref_doesnt_leak",(PyCFunction)test_xdecref_doesnt_leak,      METH_NOARGS},
+    {"test_decref_doesnt_leak", (PyCFunction)test_decref_doesnt_leak,      METH_NOARGS},
+    {"test_incref_decref_API",  (PyCFunction)test_incref_decref_API,       METH_NOARGS},
     {"test_long_and_overflow", (PyCFunction)test_long_and_overflow,
      METH_NOARGS},
     {"test_long_as_double",     (PyCFunction)test_long_as_double,METH_NOARGS},
@@ -2592,12 +3915,24 @@
     {"test_empty_argparse", (PyCFunction)test_empty_argparse,METH_NOARGS},
     {"parse_tuple_and_keywords", parse_tuple_and_keywords, METH_VARARGS},
     {"test_null_strings",       (PyCFunction)test_null_strings,  METH_NOARGS},
+#ifndef PYPY_VERSION
     {"test_string_from_format", (PyCFunction)test_string_from_format, METH_NOARGS},
+#endif
     {"test_with_docstring", (PyCFunction)test_with_docstring, METH_NOARGS,
      PyDoc_STR("This is a pretty normal docstring.")},
     {"test_string_to_double", (PyCFunction)test_string_to_double, METH_NOARGS},
     {"test_unicode_compare_with_ascii", (PyCFunction)test_unicode_compare_with_ascii, METH_NOARGS},
     {"test_capsule", (PyCFunction)test_capsule, METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"test_from_contiguous", (PyCFunction)test_from_contiguous, METH_NOARGS},
+#endif
+#if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION)
+    {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS},
+#endif
+    {"getbuffer_with_null_view", getbuffer_with_null_view, METH_O},
+    {"test_buildvalue_N",       test_buildvalue_N,               METH_NOARGS},
+    {"get_args", get_args, METH_VARARGS},
+    {"get_kwargs", (PyCFunction)get_kwargs, METH_VARARGS|METH_KEYWORDS},
     {"getargs_tuple",           getargs_tuple,                   METH_VARARGS},
     {"getargs_keywords", (PyCFunction)getargs_keywords,
       METH_VARARGS|METH_KEYWORDS},
@@ -2621,7 +3956,14 @@
         (PyCFunction)test_long_long_and_overflow, METH_NOARGS},
     {"test_L_code",             (PyCFunction)test_L_code,        METH_NOARGS},
 #endif
+    {"getargs_f",               getargs_f,                       METH_VARARGS},
+    {"getargs_d",               getargs_d,                       METH_VARARGS},
+    {"getargs_D",               getargs_D,                       METH_VARARGS},
+    {"getargs_S",               getargs_S,                       METH_VARARGS},
+    {"getargs_Y",               getargs_Y,                       METH_VARARGS},
+    {"getargs_U",               getargs_U,                       METH_VARARGS},
     {"getargs_c",               getargs_c,                       METH_VARARGS},
+    {"getargs_C",               getargs_C,                       METH_VARARGS},
     {"getargs_s",               getargs_s,                       METH_VARARGS},
     {"getargs_s_star",          getargs_s_star,                  METH_VARARGS},
     {"getargs_s_hash",          getargs_s_hash,                  METH_VARARGS},
@@ -2636,6 +3978,10 @@
     {"getargs_Z",               getargs_Z,                       METH_VARARGS},
     {"getargs_Z_hash",          getargs_Z_hash,                  METH_VARARGS},
     {"getargs_w_star",          getargs_w_star,                  METH_VARARGS},
+    {"getargs_es",              getargs_es,                      METH_VARARGS},
+    {"getargs_et",              getargs_et,                      METH_VARARGS},
+    {"getargs_es_hash",         getargs_es_hash,                 METH_VARARGS},
+    {"getargs_et_hash",         getargs_et_hash,                 METH_VARARGS},
     {"codec_incrementalencoder",
      (PyCFunction)codec_incrementalencoder,                      METH_VARARGS},
     {"codec_incrementaldecoder",
@@ -2646,6 +3992,10 @@
     {"test_widechar",           (PyCFunction)test_widechar,      METH_NOARGS},
     {"unicode_aswidechar",      unicode_aswidechar,              METH_VARARGS},
     {"unicode_aswidecharstring",unicode_aswidecharstring,        METH_VARARGS},
+#ifndef PYPY_VERSION
+    {"unicode_asucs4",          unicode_asucs4,                  METH_VARARGS},
+    {"unicode_copycharacters",  unicode_copycharacters,          METH_VARARGS},
+#endif
     {"unicode_encodedecimal",   unicode_encodedecimal,           METH_VARARGS},
     {"unicode_transformdecimaltoascii", unicode_transformdecimaltoascii, METH_VARARGS},
     {"unicode_legacy_string",   unicode_legacy_string,           METH_VARARGS},
@@ -2671,11 +4021,81 @@
     {"pytime_object_to_time_t", test_pytime_object_to_time_t,  METH_VARARGS},
     {"pytime_object_to_timeval", test_pytime_object_to_timeval,  METH_VARARGS},
     {"pytime_object_to_timespec", test_pytime_object_to_timespec,  METH_VARARGS},
+    {"with_tp_del",             with_tp_del,                     METH_VARARGS},
 #endif
+    {"create_cfunction",        create_cfunction,                METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"test_pymem_alloc0",
+     (PyCFunction)test_pymem_alloc0, METH_NOARGS},
+    {"test_pymem_setrawallocators",
+     (PyCFunction)test_pymem_setrawallocators, METH_NOARGS},
+    {"test_pymem_setallocators",
+     (PyCFunction)test_pymem_setallocators, METH_NOARGS},
+    {"test_pyobject_setallocators",
+     (PyCFunction)test_pyobject_setallocators, METH_NOARGS},
+#endif
+    {"no_docstring",
+        (PyCFunction)test_with_docstring, METH_NOARGS},
+    {"docstring_empty",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_empty},
+    {"docstring_no_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_no_signature},
+    {"docstring_with_invalid_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_invalid_signature},
+    {"docstring_with_invalid_signature2",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_invalid_signature2},
+    {"docstring_with_signature",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature},
+    {"docstring_with_signature_but_no_doc",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_but_no_doc},
+    {"docstring_with_signature_and_extra_newlines",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_and_extra_newlines},
+    {"docstring_with_signature_with_defaults",
+        (PyCFunction)test_with_docstring, METH_NOARGS,
+        docstring_with_signature_with_defaults},
+    {"raise_signal",
+     (PyCFunction)test_raise_signal, METH_VARARGS},
 #ifdef WITH_THREAD
     {"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_O,
      PyDoc_STR("set_error_class(error_class) -> None")},
 #endif
+#ifndef PYPY_VERSION
+    {"pymarshal_write_long_to_file",
+        pymarshal_write_long_to_file, METH_VARARGS},
+    {"pymarshal_write_object_to_file",
+        pymarshal_write_object_to_file, METH_VARARGS},
+    {"pymarshal_read_short_from_file",
+        pymarshal_read_short_from_file, METH_VARARGS},
+    {"pymarshal_read_long_from_file",
+        pymarshal_read_long_from_file, METH_VARARGS},
+    {"pymarshal_read_last_object_from_file",
+        pymarshal_read_last_object_from_file, METH_VARARGS},
+    {"pymarshal_read_object_from_file",
+        pymarshal_read_object_from_file, METH_VARARGS},
+#endif
+    {"return_null_without_error",
+        return_null_without_error, METH_NOARGS},
+    {"return_result_with_error",
+        return_result_with_error, METH_NOARGS},
+#ifndef PYPY_VERSION
+    {"PyTime_FromSeconds", test_pytime_fromseconds,  METH_VARARGS},
+    {"PyTime_FromSecondsObject", test_pytime_fromsecondsobject,  METH_VARARGS},
+    {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS},
+    {"PyTime_AsTimeval", test_PyTime_AsTimeval, METH_VARARGS},
+#ifdef HAVE_CLOCK_GETTIME
+    {"PyTime_AsTimespec", test_PyTime_AsTimespec, METH_VARARGS},
+#endif
+    {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
+    {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
+    {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
+#endif
     {NULL, NULL} /* sentinel */
 };
 
@@ -2835,6 +4255,201 @@
 };
 
 
+typedef struct {
+    PyObject_HEAD
+} matmulObject;
+
+static PyObject *
+matmulType_matmul(PyObject *self, PyObject *other)
+{
+    return Py_BuildValue("(sOO)", "matmul", self, other);
+}
+
+static PyObject *
+matmulType_imatmul(PyObject *self, PyObject *other)
+{
+    return Py_BuildValue("(sOO)", "imatmul", self, other);
+}
+
+static void
+matmulType_dealloc(PyObject *self)
+{
+    Py_TYPE(self)->tp_free(self);
+}
+
+static PyNumberMethods matmulType_as_number = {
+    0,                          /* nb_add */
+    0,                          /* nb_subtract */
+    0,                          /* nb_multiply */
+    0,                          /* nb_remainde r*/
+    0,                          /* nb_divmod */
+    0,                          /* nb_power */
+    0,                          /* nb_negative */
+    0,                          /* tp_positive */
+    0,                          /* tp_absolute */
+    0,                          /* tp_bool */
+    0,                          /* nb_invert */
+    0,                          /* nb_lshift */
+    0,                          /* nb_rshift */
+    0,                          /* nb_and */
+    0,                          /* nb_xor */
+    0,                          /* nb_or */
+    0,                          /* nb_int */
+    0,                          /* nb_reserved */
+    0,                          /* nb_float */
+    0,                          /* nb_inplace_add */
+    0,                          /* nb_inplace_subtract */
+    0,                          /* nb_inplace_multiply */
+    0,                          /* nb_inplace_remainder */
+    0,                          /* nb_inplace_power */
+    0,                          /* nb_inplace_lshift */
+    0,                          /* nb_inplace_rshift */
+    0,                          /* nb_inplace_and */
+    0,                          /* nb_inplace_xor */
+    0,                          /* nb_inplace_or */
+    0,                          /* nb_floor_divide */
+    0,                          /* nb_true_divide */
+    0,                          /* nb_inplace_floor_divide */
+    0,                          /* nb_inplace_true_divide */
+    0,                          /* nb_index */
+    matmulType_matmul,        /* nb_matrix_multiply */
+    matmulType_imatmul        /* nb_matrix_inplace_multiply */
+};
+
+static PyTypeObject matmulType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "matmulType",
+    sizeof(matmulObject),               /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    matmulType_dealloc,                 /* destructor tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    0,                                  /* tp_reserved */
+    0,                                  /* tp_repr */
+    &matmulType_as_number,              /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    PyObject_GenericGetAttr,            /* tp_getattro */
+    PyObject_GenericSetAttr,            /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    0,                                  /* tp_flags */
+    "C level type with matrix operations defined",
+    0,                                  /* traverseproc tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    PyType_GenericNew,                  /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
+
+typedef struct {
+    PyObject_HEAD
+    PyObject *ao_iterator;
+} awaitObject;
+
+
+static PyObject *
+awaitObject_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+{
+    PyObject *v;
+    awaitObject *ao;
+
+    if (!PyArg_UnpackTuple(args, "awaitObject", 1, 1, &v))
+        return NULL;
+
+    ao = (awaitObject *)type->tp_alloc(type, 0);
+    if (ao == NULL) {
+        return NULL;
+    }
+
+    Py_INCREF(v);
+    ao->ao_iterator = v;
+
+    return (PyObject *)ao;
+}
+
+
+static void
+awaitObject_dealloc(awaitObject *ao)
+{
+    Py_CLEAR(ao->ao_iterator);
+    Py_TYPE(ao)->tp_free(ao);
+}
+
+
+static PyObject *
+awaitObject_await(awaitObject *ao)
+{
+    Py_INCREF(ao->ao_iterator);
+    return ao->ao_iterator;
+}
+
+static PyAsyncMethods awaitType_as_async = {
+    (unaryfunc)awaitObject_await,           /* am_await */
+    0,                                      /* am_aiter */
+    0                                       /* am_anext */
+};
+
+
+static PyTypeObject awaitType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
+    "awaitType",
+    sizeof(awaitObject),                /* tp_basicsize */
+    0,                                  /* tp_itemsize */
+    (destructor)awaitObject_dealloc,    /* destructor tp_dealloc */
+    0,                                  /* tp_print */
+    0,                                  /* tp_getattr */
+    0,                                  /* tp_setattr */
+    &awaitType_as_async,                /* tp_as_async */
+    0,                                  /* tp_repr */
+    0,                                  /* tp_as_number */
+    0,                                  /* tp_as_sequence */
+    0,                                  /* tp_as_mapping */
+    0,                                  /* tp_hash */
+    0,                                  /* tp_call */
+    0,                                  /* tp_str */
+    PyObject_GenericGetAttr,            /* tp_getattro */
+    PyObject_GenericSetAttr,            /* tp_setattro */
+    0,                                  /* tp_as_buffer */
+    0,                                  /* tp_flags */
+    "C level type with tp_as_async",
+    0,                                  /* traverseproc tp_traverse */
+    0,                                  /* tp_clear */
+    0,                                  /* tp_richcompare */
+    0,                                  /* tp_weaklistoffset */
+    0,                                  /* tp_iter */
+    0,                                  /* tp_iternext */
+    0,                                  /* tp_methods */
+    0,                                  /* tp_members */
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    0,
+    awaitObject_new,                    /* tp_new */
+    PyObject_Del,                       /* tp_free */
+};
+
 
 static struct PyModuleDef _testcapimodule = {
     PyModuleDef_HEAD_INIT,
@@ -2848,6 +4463,9 @@
     NULL
 };
 
+/* Per PEP 489, this module will not be converted to multi-phase initialization
+ */
+
 PyMODINIT_FUNC
 PyInit__testcapi(void)
 {
@@ -2856,7 +4474,7 @@
     m = PyModule_Create(&_testcapimodule);
     if (m == NULL)
         return NULL;
-    
+
     Py_TYPE(&_HashInheritanceTester_Type)=&PyType_Type;
 
     Py_TYPE(&test_structmembersType)=&PyType_Type;
@@ -2864,6 +4482,15 @@
     /* don't use a name starting with "test", since we don't want
        test_capi to automatically call this */
     PyModule_AddObject(m, "_test_structmembersType", (PyObject *)&test_structmembersType);
+    if (PyType_Ready(&matmulType) < 0)
+        return NULL;
+    Py_INCREF(&matmulType);
+    PyModule_AddObject(m, "matmulType", (PyObject *)&matmulType);
+
+    if (PyType_Ready(&awaitType) < 0)
+        return NULL;
+    Py_INCREF(&awaitType);
+    PyModule_AddObject(m, "awaitType", (PyObject *)&awaitType);
 
     PyModule_AddObject(m, "CHAR_MAX", PyLong_FromLong(CHAR_MAX));
     PyModule_AddObject(m, "CHAR_MIN", PyLong_FromLong(CHAR_MIN));
@@ -2890,6 +4517,8 @@
     Py_INCREF(&PyInstanceMethod_Type);
     PyModule_AddObject(m, "instancemethod", (PyObject *)&PyInstanceMethod_Type);
 
+    PyModule_AddIntConstant(m, "the_number_three", 3);
+
     TestError = PyErr_NewException("_testcapi.error", NULL, NULL);
     Py_INCREF(TestError);
     PyModule_AddObject(m, "error", TestError);
diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.11.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"

From pypy.commits at gmail.com  Fri Jun  2 06:51:43 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:43 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: backout merge of
 cffi-complex from release
Message-ID: <5931433f.4eacdf0a.12a2a.0f07@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91488:191e30749c0d
Date: 2017-06-02 13:29 +0300
http://bitbucket.org/pypy/pypy/changeset/191e30749c0d/

Log:	backout merge of cffi-complex from release

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.10.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
+#ifndef _CFFI_USE_EMBEDDING
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,11 +105,8 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
-PRIM_FLOATCOMPLEX  = 48
-PRIM_DOUBLECOMPLEX = 49
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 48
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +128,6 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
-    'float _Complex':     PRIM_FLOATCOMPLEX,
-    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,7 +16,6 @@
 except ImportError:
     lock = None
 
-CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -259,21 +258,15 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = []
-        csourcelines.append('# 1 "<cdef automatic initialization code>"')
-        for typename in typenames:
-            csourcelines.append('typedef int %s;' % typename)
+        csourcelines = ['typedef int %s;' % typename for typename in typenames]
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
-        # this forces pycparser to consider the following in the file
-        # called <cdef source string> from line 1
-        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        fullcsource = '\n'.join(csourcelines)
+        csource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(fullcsource)
+            ast = _get_parser().parse(csource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -283,17 +276,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
-        # and interpret that as a line number.  This will not work if
-        # the user gives explicit ``# NUM "FILE"`` directives.
+        # xxx look for ":NUM:" at the start of str(e) and try to interpret
+        # it as a line number
         line = None
         msg = str(e)
-        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
-        if match:
-            linenum = int(match.group(1), 10)
-            csourcelines = csource.splitlines()
-            if 1 <= linenum <= len(csourcelines):
-                line = csourcelines[linenum-1]
+        if msg.startswith(':') and ':' in msg[1:]:
+            linenum = msg[1:msg.find(':',1)]
+            if linenum.isdigit():
+                linenum = int(linenum, 10)
+                csourcelines = csource.splitlines()
+                if 1 <= linenum <= len(csourcelines):
+                    line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -328,12 +321,10 @@
                 break
         else:
             assert 0
-        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
-                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -357,13 +348,7 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unexpected <%s>: this construct is valid "
-                                    "C but not valid in cdef()" %
-                                    decl.__class__.__name__, decl)
-        except CDefError as e:
-            if len(e.args) == 1:
-                e.args = e.args + (current_decl,)
-            raise
+                    raise CDefError("unrecognized construct", decl)
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,13 +5,10 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            current_decl = self.args[1]
-            filename = current_decl.coord.file
-            linenum = current_decl.coord.line
-            prefix = '%s:%d: ' % (filename, linenum)
+            line = 'line %d: ' % (self.args[1].coord.line,)
         except (AttributeError, TypeError, IndexError):
-            prefix = ''
-        return '%s%s' % (prefix, self.args[0])
+            line = ''
+        return '%s%s' % (line, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,7 +6,6 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
-    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -16,7 +15,6 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
-    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -115,13 +113,3 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
-
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,8 +95,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def is_complex_type(self):
-        return False
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -117,8 +116,6 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
-        'float _Complex':     'j',
-        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -166,8 +163,6 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
-    def is_complex_type(self):
-        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,10 +79,8 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
-#define _CFFI_PRIM_FLOATCOMPLEX 48
-#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         48
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
+        if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,10 +524,8 @@
                                                     tovar, errcode)
             return
         #
-        elif (isinstance(tp, model.StructOrUnionOrEnum) or
-              isinstance(tp, model.BasePrimitiveType)):
-            # a struct (not a struct pointer) as a function argument;
-            # or, a complex (the same code works)
+        elif isinstance(tp, model.StructOrUnionOrEnum):
+            # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -572,7 +570,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double' and not tp.is_complex_type():
+            elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -736,26 +734,21 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.  We also do that for
-        # complex args and return type.
-        def need_indirection(type):
-            return (isinstance(type, model.StructOrUnion) or
-                    (isinstance(type, model.PrimitiveType) and
-                     type.is_complex_type()))
+        # arg that is a pointer to the result.
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if need_indirection(type):
+            if isinstance(type, model.StructOrUnion):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if need_indirection(tp_result):
+        if isinstance(tp_result, model.StructOrUnion):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1187,7 +1180,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
+        prnt('  { "%s", %s };' % (name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1486,12 +1479,6 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
-                if compiler_verbose:
-                    if tmpdir == '.':
-                        msg = 'the current directory is'
-                    else:
-                        msg = 'setting the current directory to'
-                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,6 +26,16 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
+
 
 class Verifier(object):
 
@@ -116,7 +126,7 @@
         return basename
 
     def get_extension(self):
-        ffiplatform._hack_at_distutils() # backward compatibility hack
+        _hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,4 @@
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8
 
-.. branch: cffi-complex
 
-Part of the upgrade to cffi 1.11
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.10.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,11 +91,6 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
-    def complex(self):
-        with self as ptr:
-            w_result = self.ctype.complex(ptr)
-        return w_result
-
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -410,13 +405,6 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
-    def write_raw_complex_data(self, real, imag):
-        with self as ptr:
-            halfsize = self.ctype.size >> 1
-            ptr2 = rffi.ptradd(ptr, halfsize)
-            misc.write_raw_float_data(ptr, real, halfsize)
-            misc.write_raw_float_data(ptr2, imag, halfsize)
-
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -658,7 +646,6 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
-    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,10 +105,8 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
-PRIM_FLOATCOMPLEX  = 48
-PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 48
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,21 +212,18 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-
-def _notimplemented_ffi_type(self, is_result_type, extra=''):
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    space = self.space
+    if self.size < 0:
+        raise oefmt(space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(self.space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s%s",
-                self.name, self.size, place, extra)
-
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    if self.size < 0:
-        raise oefmt(self.space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
-    raise _notimplemented_ffi_type(self, is_result_type)
+    raise oefmt(space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s",
+                self.name, self.size, place)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -263,13 +260,6 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
-def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
-    raise _notimplemented_ffi_type(self, is_result_type,
-        extra = " (the support for complex types inside libffi "
-                "is mostly missing at this point, so CFFI only "
-                "supports complex types as arguments or return "
-                "value in API-mode functions)")
-
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -286,7 +276,6 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
-W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,14 +73,6 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
-    def complex(self, cdata):
-        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
-        # calling complex(), just like <cdata 'int'> cannot be directly
-        # converted by calling float()
-        space = self.space
-        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
-                    self.name)
-
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,51 +532,3 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
-
-
-class W_CTypePrimitiveComplex(W_CTypePrimitive):
-    _attrs_ = []
-
-    def cast(self, w_ob):
-        space = self.space
-        if isinstance(w_ob, cdataobj.W_CData):
-            if not isinstance(w_ob.ctype, W_CTypePrimitive):
-                raise oefmt(space.w_TypeError,
-                            "cannot cast ctype '%s' to ctype '%s'",
-                            w_ob.ctype.name, self.name)
-            w_ob = w_ob.convert_to_object()
-        #
-        imag = 0.0
-        if space.isinstance_w(w_ob, space.w_bytes):
-            real = self.cast_str(w_ob)
-        elif space.isinstance_w(w_ob, space.w_unicode):
-            real = self.cast_unicode(w_ob)
-        else:
-            real, imag = space.unpackcomplex(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self)
-        w_cdata.write_raw_complex_data(real, imag)
-        return w_cdata
-
-    def complex(self, cdata):
-        return self.convert_to_object(cdata)
-
-    def convert_to_object(self, cdata):
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        real = misc.read_raw_float_data(cdata, halfsize)
-        imag = misc.read_raw_float_data(cdata2, halfsize)
-        return self.space.newcomplex(real, imag)
-
-    def convert_from_object(self, cdata, w_ob):
-        space = self.space
-        real, imag = space.unpackcomplex(w_ob)
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        misc.write_raw_float_data(cdata,  real, halfsize)
-        misc.write_raw_float_data(cdata2, imag, halfsize)
-
-    def nonzero(self, cdata):
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        return (misc.is_nonnull_float(cdata, halfsize) |
-                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls, rep=1):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
+def eptype(name, TYPE, ctypecls):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,9 +94,6 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
-eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
-eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
-
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,7 +8,6 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
-from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -71,8 +70,6 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
-        "float _Complex",
-        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -212,7 +209,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.  (Same with complex numbers.)
+        # trampoline functions for PyPy.
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -221,13 +218,11 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
-                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
+                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
-                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
+            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    TOK__COMPLEX,
+    //TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,7 +171,6 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
-        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -614,7 +613,6 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
-    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -670,8 +668,6 @@
         break;
     }
 
-    t1complex = 0;
-
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -682,7 +678,6 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
-        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -736,11 +731,9 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
-            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
-            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -807,13 +800,6 @@
         }
         next_token(tok);
     }
-    if (tok->kind == TOK__COMPLEX)
-    {
-        if (t1complex == 0)
-            return parse_error(tok, "_Complex type combination unsupported");
-        t1 = t1complex;
-        next_token(tok);
-    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,10 +78,8 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
-#define _CFFI_PRIM_FLOATCOMPLEX 48
-#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         48
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -174,56 +174,37 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
+    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type(name + " _Complex")
-        assert bool(cast(p, 0)) is False
+        p = new_primitive_type("_Complex " + name)
+        assert bool(cast(p, 0))
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j)) is False
+        assert bool(cast(p, 0j))
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
-        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
-        assert complex(cast(p, -INF)) == -INF
+        assert float(cast(p, INF*1j)) == INF*1j
+        assert float(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
+            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
 
-        assert cast(p, -1.1j) == cast(p, -1.1j)
+        assert cast(p, -1.1j) != cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
-        assert complex(cast(p, b'\x09')) == 9.0 + 0j
-        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
-        assert complex(cast(p, True)) == 1.0 + 0j
+        assert repr(complex(cast(p, -0j))) == '-0j'
+        assert complex(cast(p, '\x09')) == 9.0
+        assert complex(cast(p, True)) == 1.0
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
-        #
-        for basetype in ["char", "int", "uint64_t", "float",
-                         "double", "long double"]:
-            baseobj = cast(new_primitive_type(basetype), 65)
-            py.test.raises(TypeError, complex, baseobj)
-        #
-        BArray = new_array_type(new_pointer_type(p), 10)
-        x = newp(BArray, None)
-        x[5] = 12.34 + 56.78j
-        assert type(x[5]) is complex
-        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
-        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
-        #
-        class Foo:
-            def __complex__(self):
-                return 2 + 3j
-        assert complex(Foo()) == 2 + 3j
-        assert complex(cast(p, Foo())) == 2 + 3j
-    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
+        py.test.raises(cast, new_primitive_type(name), 1+2j)
+    py.test.raises(cast, new_primitive_type("int"), 1+2j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1124,34 +1105,6 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
-def test_call_function_24():
-    BFloat = new_primitive_type("float")
-    BFloatComplex = new_primitive_type("float _Complex")
-    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
-    if 0:   # libffi returning nonsense silently, so logic disabled for now
-        f = cast(BFunc3, _testfunc(24))
-        result = f(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-    else:
-        f = cast(BFunc3, _testfunc(9))
-        py.test.raises(NotImplementedError, f, 12.3, 34.5)
-
-def test_call_function_25():
-    BDouble = new_primitive_type("double")
-    BDoubleComplex = new_primitive_type("double _Complex")
-    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
-    if 0:   # libffi returning nonsense silently, so logic disabled for now
-        f = cast(BFunc3, _testfunc(25))
-        result = f(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
-    else:
-        f = cast(BFunc3, _testfunc(9))
-        py.test.raises(NotImplementedError, f, 12.3, 34.5)
-
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
@@ -3843,7 +3796,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,8 +148,6 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
-            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
-            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -275,11 +273,6 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
-    #
-    parse_error("_Complex", "identifier expected", 0)
-    parse_error("int _Complex", "_Complex type combination unsupported", 4)
-    parse_error("long double _Complex", "_Complex type combination unsupported",
-                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,68 +1819,6 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
-    def test_function_returns_float_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "float _Complex f1(float a, float b);",
-            "test_function_returns_float_complex", """
-            #include <complex.h>
-            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-        """, min_version=(1, 11, 0))
-        result = lib.f1(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-
-    def test_function_returns_double_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "double _Complex f1(double a, double b);",
-            "test_function_returns_double_complex", """
-            #include <complex.h>
-            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-        """, min_version=(1, 11, 0))
-        result = lib.f1(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert result.imag == 2*5.1  # exact
-
-    def test_function_argument_float_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "float f1(float _Complex x);",
-            "test_function_argument_float_complex", """
-            #include <complex.h>
-            static float f1(float _Complex x) { return cabsf(x); }
-        """, min_version=(1, 11, 0))
-        x = complex(12.34, 56.78)
-        result = lib.f1(x)
-        assert abs(result - abs(x)) < 1e-5
-        result2 = lib.f1(ffi.cast("float _Complex", x))
-        assert result2 == result
-
-    def test_function_argument_double_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "double f1(double _Complex);",
-            "test_function_argument_double_complex", """
-            #include <complex.h>
-            static double f1(double _Complex x) { return cabs(x); }
-        """, min_version=(1, 11, 0))
-        x = complex(12.34, 56.78)
-        result = lib.f1(x)
-        assert abs(result - abs(x)) < 1e-11
-        result2 = lib.f1(ffi.cast("double _Complex", x))
-        assert result2 == result
-
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,7 +8,6 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
-from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -84,9 +83,8 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                ctyperesptr = w_result_cdata.ctype
-                assert isinstance(ctyperesptr, W_CTypePointer)
-                return w_result_cdata._do_getitem(ctyperesptr, 0)
+                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
+                return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -111,14 +109,13 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
-    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union/complex>
+        farg = fargs[i]      # <ptr to struct/union>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,27 +229,13 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == (
-           "<cdef source string>:1: foo: a function with only '(...)' "
-           "as argument is not correct C")
+    assert str(e.value) == \
+           "foo: a function with only '(...)' as argument is not correct C"
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert str(e.value).startswith(
-        'cannot parse "x y z"\n<cdef source string>:1:')
-    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
-    assert str(e.value).startswith(
-        'cannot parse "x y z"\n<cdef source string>:4:')
-
-def test_error_custom_lineno():
-    ffi = FFI()
-    e = py.test.raises(CDefError, ffi.cdef, """
-# 42 "foobar"
-
-    a b c d
-    """)
-    assert str(e.value).startswith('parse error\nfoobar:43:')
+    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -293,8 +279,7 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
-                            " unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -452,9 +437,3 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
-
-def test_error_invalid_syntax_for_cdef():
-    ffi = FFI()
-    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
-    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
-                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,18 +240,15 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
-        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert X == (typename in ('float _Complex', 'double _Complex'))
-        assert I + F + C + X == 1      # one and only one of them is true
+        assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,8 +1705,6 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
-            'float _Complex',
-            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,8 +156,6 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
-            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
-            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -283,11 +281,6 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
-    #
-    parse_error("_Complex", "identifier expected", 0)
-    parse_error("int _Complex", "_Complex type combination unsupported", 4)
-    parse_error("long double _Complex", "_Complex type combination unsupported",
-                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,6 +48,7 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
+
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,8 +1553,7 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
-        b"but no code was attached "
+        b"extern \"Python\": function bar() called, but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2002,60 +2001,6 @@
     """)
     assert lib.f1(52).a == 52
 
-def test_function_returns_float_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("float _Complex f1(float a, float b);");
-    lib = verify(ffi, "test_function_returns_float_complex", """
-        #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
-    result = lib.f1(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-
-def test_function_returns_double_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("double _Complex f1(double a, double b);");
-    lib = verify(ffi, "test_function_returns_double_complex", """
-        #include <complex.h>
-        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
-    result = lib.f1(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert result.imag == 2*5.1  # exact
-
-def test_function_argument_float_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("float f1(float _Complex x);");
-    lib = verify(ffi, "test_function_argument_float_complex", """
-        #include <complex.h>
-        static float f1(float _Complex x) { return cabsf(x); }
-    """)
-    x = complex(12.34, 56.78)
-    result = lib.f1(x)
-    assert abs(result - abs(x)) < 1e-5
-
-def test_function_argument_double_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("double f1(double _Complex);");
-    lib = verify(ffi, "test_function_argument_double_complex", """
-        #include <complex.h>
-        static double f1(double _Complex x) { return cabs(x); }
-    """)
-    x = complex(12.34, 56.78)
-    result = lib.f1(x)
-    assert abs(result - abs(x)) < 1e-11
-
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,18 +220,15 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
-        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert X == (typename in ('float _Complex', 'double _Complex'))
-        assert I + F + C + X == 1      # one and only one of them is true
+        assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Fri Jun  2 06:51:50 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:50 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: backout merge of
 cffi-complex from release
Message-ID: <59314346.d8aadf0a.fd8aa.da6e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91491:0eefaf66cee9
Date: 2017-06-02 13:40 +0300
http://bitbucket.org/pypy/pypy/changeset/0eefaf66cee9/

Log:	backout merge of cffi-complex from release (grafted from
	191e30749c0d2b86c8a1d6400e5ac2bc238bb176)

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.11.0
+Version: 1.10.0
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.11.0"
-__version_info__ = (1, 11, 0)
+__version__ = "1.10.0"
+__version_info__ = (1, 10, 0)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
+#ifndef _CFFI_USE_EMBEDDING
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -233,7 +233,7 @@
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
-                               "\ncompiled with cffi version: 1.11.0"
+                               "\ncompiled with cffi version: 1.10.0"
                                "\n_cffi_backend module: ", f);
             modules = PyImport_GetModuleDict();
             mod = PyDict_GetItemString(modules, "_cffi_backend");
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -105,11 +105,8 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
-PRIM_FLOATCOMPLEX  = 48
-PRIM_DOUBLECOMPLEX = 49
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 48
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +128,6 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
-    'float _Complex':     PRIM_FLOATCOMPLEX,
-    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
     'int8_t':             PRIM_INT8,
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,7 +16,6 @@
 except ImportError:
     lock = None
 
-CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -259,21 +258,15 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = []
-        csourcelines.append('# 1 "<cdef automatic initialization code>"')
-        for typename in typenames:
-            csourcelines.append('typedef int %s;' % typename)
+        csourcelines = ['typedef int %s;' % typename for typename in typenames]
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
-        # this forces pycparser to consider the following in the file
-        # called <cdef source string> from line 1
-        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        fullcsource = '\n'.join(csourcelines)
+        csource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(fullcsource)
+            ast = _get_parser().parse(csource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -283,17 +276,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
-        # and interpret that as a line number.  This will not work if
-        # the user gives explicit ``# NUM "FILE"`` directives.
+        # xxx look for ":NUM:" at the start of str(e) and try to interpret
+        # it as a line number
         line = None
         msg = str(e)
-        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
-        if match:
-            linenum = int(match.group(1), 10)
-            csourcelines = csource.splitlines()
-            if 1 <= linenum <= len(csourcelines):
-                line = csourcelines[linenum-1]
+        if msg.startswith(':') and ':' in msg[1:]:
+            linenum = msg[1:msg.find(':',1)]
+            if linenum.isdigit():
+                linenum = int(linenum, 10)
+                csourcelines = csource.splitlines()
+                if 1 <= linenum <= len(csourcelines):
+                    line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -328,12 +321,10 @@
                 break
         else:
             assert 0
-        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
-                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -357,13 +348,7 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unexpected <%s>: this construct is valid "
-                                    "C but not valid in cdef()" %
-                                    decl.__class__.__name__, decl)
-        except CDefError as e:
-            if len(e.args) == 1:
-                e.args = e.args + (current_decl,)
-            raise
+                    raise CDefError("unrecognized construct", decl)
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,13 +5,10 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            current_decl = self.args[1]
-            filename = current_decl.coord.file
-            linenum = current_decl.coord.line
-            prefix = '%s:%d: ' % (filename, linenum)
+            line = 'line %d: ' % (self.args[1].coord.line,)
         except (AttributeError, TypeError, IndexError):
-            prefix = ''
-        return '%s%s' % (prefix, self.args[0])
+            line = ''
+        return '%s%s' % (line, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,7 +6,6 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
-    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -16,7 +15,6 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
-    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -115,13 +113,3 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
-
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -95,8 +95,7 @@
 
 
 class BasePrimitiveType(BaseType):
-    def is_complex_type(self):
-        return False
+    pass
 
 
 class PrimitiveType(BasePrimitiveType):
@@ -117,8 +116,6 @@
         'float':              'f',
         'double':             'f',
         'long double':        'f',
-        'float _Complex':     'j',
-        'double _Complex':    'j',
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
@@ -166,8 +163,6 @@
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'i'
     def is_float_type(self):
         return self.ALL_PRIMITIVE_TYPES[self.name] == 'f'
-    def is_complex_type(self):
-        return self.ALL_PRIMITIVE_TYPES[self.name] == 'j'
 
     def build_backend_type(self, ffi, finishlist):
         return global_cache(self, ffi, 'new_primitive_type', self.name)
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -79,10 +79,8 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
-#define _CFFI_PRIM_FLOATCOMPLEX 48
-#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         48
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -506,7 +506,7 @@
 
     def _convert_funcarg_to_c(self, tp, fromvar, tovar, errcode):
         extraarg = ''
-        if isinstance(tp, model.BasePrimitiveType) and not tp.is_complex_type():
+        if isinstance(tp, model.BasePrimitiveType):
             if tp.is_integer_type() and tp.name != '_Bool':
                 converter = '_cffi_to_c_int'
                 extraarg = ', %s' % tp.name
@@ -524,10 +524,8 @@
                                                     tovar, errcode)
             return
         #
-        elif (isinstance(tp, model.StructOrUnionOrEnum) or
-              isinstance(tp, model.BasePrimitiveType)):
-            # a struct (not a struct pointer) as a function argument;
-            # or, a complex (the same code works)
+        elif isinstance(tp, model.StructOrUnionOrEnum):
+            # a struct (not a struct pointer) as a function argument
             self._prnt('  if (_cffi_to_c((char *)&%s, _cffi_type(%d), %s) < 0)'
                       % (tovar, self._gettypenum(tp), fromvar))
             self._prnt('    %s;' % errcode)
@@ -572,7 +570,7 @@
                 return '_cffi_from_c_int(%s, %s)' % (var, tp.name)
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
-            elif tp.name != 'long double' and not tp.is_complex_type():
+            elif tp.name != 'long double':
                 return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
@@ -736,26 +734,21 @@
         #
         # the PyPy version: need to replace struct/union arguments with
         # pointers, and if the result is a struct/union, insert a first
-        # arg that is a pointer to the result.  We also do that for
-        # complex args and return type.
-        def need_indirection(type):
-            return (isinstance(type, model.StructOrUnion) or
-                    (isinstance(type, model.PrimitiveType) and
-                     type.is_complex_type()))
+        # arg that is a pointer to the result.
         difference = False
         arguments = []
         call_arguments = []
         context = 'argument of %s' % name
         for i, type in enumerate(tp.args):
             indirection = ''
-            if need_indirection(type):
+            if isinstance(type, model.StructOrUnion):
                 indirection = '*'
                 difference = True
             arg = type.get_c_name(' %sx%d' % (indirection, i), context)
             arguments.append(arg)
             call_arguments.append('%sx%d' % (indirection, i))
         tp_result = tp.result
-        if need_indirection(tp_result):
+        if isinstance(tp_result, model.StructOrUnion):
             context = 'result of %s' % name
             arg = tp_result.get_c_name(' *result', context)
             arguments.insert(0, arg)
@@ -1187,7 +1180,7 @@
             size_of_result = '(int)sizeof(%s)' % (
                 tp.result.get_c_name('', context),)
         prnt('static struct _cffi_externpy_s _cffi_externpy__%s =' % name)
-        prnt('  { "%s.%s", %s };' % (self.module_name, name, size_of_result))
+        prnt('  { "%s", %s };' % (name, size_of_result))
         prnt()
         #
         arguments = []
@@ -1486,12 +1479,6 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
-                if compiler_verbose:
-                    if tmpdir == '.':
-                        msg = 'the current directory is'
-                    else:
-                        msg = 'setting the current directory to'
-                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,6 +26,16 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
+
 
 class Verifier(object):
 
@@ -116,7 +126,7 @@
         return basename
 
     def get_extension(self):
-        ffiplatform._hack_at_distutils() # backward compatibility hack
+        _hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -5,6 +5,4 @@
 .. this is a revision shortly after release-pypy2.7-v5.8.0
 .. startrev: 558bd00b3dd8
 
-.. branch: cffi-complex
 
-Part of the upgrade to cffi 1.11
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.11.0"
+VERSION = "1.10.0"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/cdataobj.py b/pypy/module/_cffi_backend/cdataobj.py
--- a/pypy/module/_cffi_backend/cdataobj.py
+++ b/pypy/module/_cffi_backend/cdataobj.py
@@ -91,11 +91,6 @@
             w_result = self.ctype.float(ptr)
         return w_result
 
-    def complex(self):
-        with self as ptr:
-            w_result = self.ctype.complex(ptr)
-        return w_result
-
     def len(self):
         from pypy.module._cffi_backend import ctypearray
         space = self.space
@@ -410,13 +405,6 @@
         with self as ptr:
             misc.write_raw_float_data(ptr, source, self.ctype.size)
 
-    def write_raw_complex_data(self, real, imag):
-        with self as ptr:
-            halfsize = self.ctype.size >> 1
-            ptr2 = rffi.ptradd(ptr, halfsize)
-            misc.write_raw_float_data(ptr, real, halfsize)
-            misc.write_raw_float_data(ptr2, imag, halfsize)
-
     def convert_to_object(self):
         with self as ptr:
             w_obj = self.ctype.convert_to_object(ptr)
@@ -658,7 +646,6 @@
     __int__ = interp2app(W_CData.int),
     __long__ = interp2app(W_CData.long),
     __float__ = interp2app(W_CData.float),
-    __complex__ = interp2app(W_CData.complex),
     __len__ = interp2app(W_CData.len),
     __lt__ = interp2app(W_CData.lt),
     __le__ = interp2app(W_CData.le),
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -105,10 +105,8 @@
 PRIM_UINT_FAST64   = 45
 PRIM_INTMAX        = 46
 PRIM_UINTMAX       = 47
-PRIM_FLOATCOMPLEX  = 48
-PRIM_DOUBLECOMPLEX = 49
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 48
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -20,7 +20,7 @@
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStruct, W_CTypeUnion
 from pypy.module._cffi_backend.ctypeprim import (W_CTypePrimitiveSigned,
     W_CTypePrimitiveUnsigned, W_CTypePrimitiveCharOrUniChar,
-    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble, W_CTypePrimitiveComplex)
+    W_CTypePrimitiveFloat, W_CTypePrimitiveLongDouble)
 
 
 class W_CTypeFunc(W_CTypePtrBase):
@@ -212,21 +212,18 @@
 
 # ----------
 # We attach to the classes small methods that return a 'ffi_type'
-
-def _notimplemented_ffi_type(self, is_result_type, extra=''):
+def _missing_ffi_type(self, cifbuilder, is_result_type):
+    space = self.space
+    if self.size < 0:
+        raise oefmt(space.w_TypeError,
+                    "ctype '%s' has incomplete type", self.name)
     if is_result_type:
         place = "return value"
     else:
         place = "argument"
-    raise oefmt(self.space.w_NotImplementedError,
-                "ctype '%s' (size %d) not supported as %s%s",
-                self.name, self.size, place, extra)
-
-def _missing_ffi_type(self, cifbuilder, is_result_type):
-    if self.size < 0:
-        raise oefmt(self.space.w_TypeError,
-                    "ctype '%s' has incomplete type", self.name)
-    raise _notimplemented_ffi_type(self, is_result_type)
+    raise oefmt(space.w_NotImplementedError,
+                "ctype '%s' (size %d) not supported as %s",
+                self.name, self.size, place)
 
 def _struct_ffi_type(self, cifbuilder, is_result_type):
     if self.size >= 0:
@@ -263,13 +260,6 @@
 def _primlongdouble_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_longdouble
 
-def _primcomplex_ffi_type(self, cifbuilder, is_result_type):
-    raise _notimplemented_ffi_type(self, is_result_type,
-        extra = " (the support for complex types inside libffi "
-                "is mostly missing at this point, so CFFI only "
-                "supports complex types as arguments or return "
-                "value in API-mode functions)")
-
 def _ptr_ffi_type(self, cifbuilder, is_result_type):
     return clibffi.ffi_type_pointer
 
@@ -286,7 +276,6 @@
 W_CTypePrimitiveUnsigned._get_ffi_type      = _primunsigned_ffi_type
 W_CTypePrimitiveFloat._get_ffi_type         = _primfloat_ffi_type
 W_CTypePrimitiveLongDouble._get_ffi_type    = _primlongdouble_ffi_type
-W_CTypePrimitiveComplex._get_ffi_type       = _primcomplex_ffi_type
 W_CTypePtrBase._get_ffi_type                = _ptr_ffi_type
 W_CTypeVoid._get_ffi_type                   = _void_ffi_type
 # ----------
diff --git a/pypy/module/_cffi_backend/ctypeobj.py b/pypy/module/_cffi_backend/ctypeobj.py
--- a/pypy/module/_cffi_backend/ctypeobj.py
+++ b/pypy/module/_cffi_backend/ctypeobj.py
@@ -73,14 +73,6 @@
         raise oefmt(space.w_TypeError, "float() not supported on cdata '%s'",
                     self.name)
 
-    def complex(self, cdata):
-        # <cdata 'float'> or <cdata 'int'> cannot be directly converted by
-        # calling complex(), just like <cdata 'int'> cannot be directly
-        # converted by calling float()
-        space = self.space
-        raise oefmt(space.w_TypeError, "complex() not supported on cdata '%s'",
-                    self.name)
-
     def convert_to_object(self, cdata):
         space = self.space
         raise oefmt(space.w_TypeError, "cannot return a cdata '%s'", self.name)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -532,51 +532,3 @@
     @jit.dont_look_inside
     def nonzero(self, cdata):
         return misc.is_nonnull_longdouble(cdata)
-
-
-class W_CTypePrimitiveComplex(W_CTypePrimitive):
-    _attrs_ = []
-
-    def cast(self, w_ob):
-        space = self.space
-        if isinstance(w_ob, cdataobj.W_CData):
-            if not isinstance(w_ob.ctype, W_CTypePrimitive):
-                raise oefmt(space.w_TypeError,
-                            "cannot cast ctype '%s' to ctype '%s'",
-                            w_ob.ctype.name, self.name)
-            w_ob = w_ob.convert_to_object()
-        #
-        imag = 0.0
-        if space.isinstance_w(w_ob, space.w_bytes):
-            real = self.cast_str(w_ob)
-        elif space.isinstance_w(w_ob, space.w_unicode):
-            real = self.cast_unicode(w_ob)
-        else:
-            real, imag = space.unpackcomplex(w_ob)
-        w_cdata = cdataobj.W_CDataMem(space, self)
-        w_cdata.write_raw_complex_data(real, imag)
-        return w_cdata
-
-    def complex(self, cdata):
-        return self.convert_to_object(cdata)
-
-    def convert_to_object(self, cdata):
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        real = misc.read_raw_float_data(cdata, halfsize)
-        imag = misc.read_raw_float_data(cdata2, halfsize)
-        return self.space.newcomplex(real, imag)
-
-    def convert_from_object(self, cdata, w_ob):
-        space = self.space
-        real, imag = space.unpackcomplex(w_ob)
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        misc.write_raw_float_data(cdata,  real, halfsize)
-        misc.write_raw_float_data(cdata2, imag, halfsize)
-
-    def nonzero(self, cdata):
-        halfsize = self.size >> 1
-        cdata2 = rffi.ptradd(cdata, halfsize)
-        return (misc.is_nonnull_float(cdata, halfsize) |
-                misc.is_nonnull_float(cdata2, halfsize))
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -66,8 +66,8 @@
 
 PRIMITIVE_TYPES = {}
 
-def eptype(name, TYPE, ctypecls, rep=1):
-    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE) * rep, alignment(TYPE)
+def eptype(name, TYPE, ctypecls):
+    PRIMITIVE_TYPES[name] = ctypecls, rffi.sizeof(TYPE), alignment(TYPE)
 
 def eptypesize(name, size, ctypecls):
     for TYPE in [lltype.Signed, lltype.SignedLongLong, rffi.SIGNEDCHAR,
@@ -94,9 +94,6 @@
 eptype("long double", rffi.LONGDOUBLE, ctypeprim.W_CTypePrimitiveLongDouble)
 eptype("_Bool",  lltype.Bool,          ctypeprim.W_CTypePrimitiveBool)
 
-eptype("float _Complex",  rffi.FLOAT,  ctypeprim.W_CTypePrimitiveComplex, rep=2)
-eptype("double _Complex", rffi.DOUBLE, ctypeprim.W_CTypePrimitiveComplex, rep=2)
-
 eptypesize("int8_t",   1, ctypeprim.W_CTypePrimitiveSigned)
 eptypesize("uint8_t",  1, ctypeprim.W_CTypePrimitiveUnsigned)
 eptypesize("int16_t",  2, ctypeprim.W_CTypePrimitiveSigned)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -8,7 +8,6 @@
 from pypy.module import _cffi_backend
 from pypy.module._cffi_backend.ctypeobj import W_CType
 from pypy.module._cffi_backend import cffi_opcode, newtype, ctypestruct
-from pypy.module._cffi_backend import ctypeprim
 from pypy.module._cffi_backend import parse_c_type
 
 
@@ -71,8 +70,6 @@
         "uint_fast64_t",
         "intmax_t",
         "uintmax_t",
-        "float _Complex",
-        "double _Complex",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
@@ -212,7 +209,7 @@
         # which the struct args are replaced with ptr-to- struct, and
         # a struct return value is replaced with a hidden first arg of
         # type ptr-to-struct.  This is how recompiler.py produces
-        # trampoline functions for PyPy.  (Same with complex numbers.)
+        # trampoline functions for PyPy.
         if self.nostruct_ctype is None:
             fargs, fret, ellipsis, abi = self._unpack(ffi)
             # 'locs' will be a string of the same length as the final fargs,
@@ -221,13 +218,11 @@
             locs = ['\x00'] * len(fargs)
             for i in range(len(fargs)):
                 farg = fargs[i]
-                if (isinstance(farg, ctypestruct.W_CTypeStructOrUnion) or
-                    isinstance(farg, ctypeprim.W_CTypePrimitiveComplex)):
+                if isinstance(farg, ctypestruct.W_CTypeStructOrUnion):
                     farg = newtype.new_pointer_type(ffi.space, farg)
                     fargs[i] = farg
                     locs[i] = 'A'
-            if (isinstance(fret, ctypestruct.W_CTypeStructOrUnion) or
-                isinstance(fret, ctypeprim.W_CTypePrimitiveComplex)):
+            if isinstance(fret, ctypestruct.W_CTypeStructOrUnion):
                 fret = newtype.new_pointer_type(ffi.space, fret)
                 fargs = [fret] + fargs
                 locs = ['R'] + locs
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -37,7 +37,7 @@
     /* keywords */
     TOK__BOOL,
     TOK_CHAR,
-    TOK__COMPLEX,
+    //TOK__COMPLEX,
     TOK_CONST,
     TOK_DOUBLE,
     TOK_ENUM,
@@ -171,7 +171,6 @@
         if (tok->size == 5 && !memcmp(p, "_Bool", 5))  tok->kind = TOK__BOOL;
         if (tok->size == 7 && !memcmp(p,"__cdecl",7))  tok->kind = TOK_CDECL;
         if (tok->size == 9 && !memcmp(p,"__stdcall",9))tok->kind = TOK_STDCALL;
-        if (tok->size == 8 && !memcmp(p,"_Complex",8)) tok->kind = TOK__COMPLEX;
         break;
     case 'c':
         if (tok->size == 4 && !memcmp(p, "char", 4))   tok->kind = TOK_CHAR;
@@ -614,7 +613,6 @@
 {
     unsigned int t0;
     _cffi_opcode_t t1;
-    _cffi_opcode_t t1complex;
     int modifiers_length, modifiers_sign;
 
  qualifiers:
@@ -670,8 +668,6 @@
         break;
     }
 
-    t1complex = 0;
-
     if (modifiers_length || modifiers_sign) {
 
         switch (tok->kind) {
@@ -682,7 +678,6 @@
         case TOK_STRUCT:
         case TOK_UNION:
         case TOK_ENUM:
-        case TOK__COMPLEX:
             return parse_error(tok, "invalid combination of types");
 
         case TOK_DOUBLE:
@@ -736,11 +731,9 @@
             break;
         case TOK_FLOAT:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOAT);
-            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_FLOATCOMPLEX);
             break;
         case TOK_DOUBLE:
             t1 = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLE);
-            t1complex = _CFFI_OP(_CFFI_OP_PRIMITIVE, _CFFI_PRIM_DOUBLECOMPLEX);
             break;
         case TOK_IDENTIFIER:
         {
@@ -807,13 +800,6 @@
         }
         next_token(tok);
     }
-    if (tok->kind == TOK__COMPLEX)
-    {
-        if (t1complex == 0)
-            return parse_error(tok, "_Complex type combination unsupported");
-        t1 = t1complex;
-        next_token(tok);
-    }
 
     return parse_sequel(tok, write_ds(tok, t1));
 }
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -78,10 +78,8 @@
 #define _CFFI_PRIM_UINT_FAST64  45
 #define _CFFI_PRIM_INTMAX       46
 #define _CFFI_PRIM_UINTMAX      47
-#define _CFFI_PRIM_FLOATCOMPLEX 48
-#define _CFFI_PRIM_DOUBLECOMPLEX 49
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         48
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.11.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):
@@ -174,56 +174,37 @@
         py.test.raises(TypeError, cast, p, None)
 
 def test_complex_types():
+    py.test.skip("later")
     INF = 1E200 * 1E200
     for name in ["float", "double"]:
-        p = new_primitive_type(name + " _Complex")
-        assert bool(cast(p, 0)) is False
+        p = new_primitive_type("_Complex " + name)
+        assert bool(cast(p, 0))
         assert bool(cast(p, INF))
         assert bool(cast(p, -INF))
-        assert bool(cast(p, 0j)) is False
+        assert bool(cast(p, 0j))
         assert bool(cast(p, INF*1j))
         assert bool(cast(p, -INF*1j))
-        # "can't convert complex to float", like CPython's "float(0j)"
         py.test.raises(TypeError, int, cast(p, -150))
         py.test.raises(TypeError, long, cast(p, -150))
         py.test.raises(TypeError, float, cast(p, -150))
         assert complex(cast(p, 1.25)) == 1.25
         assert complex(cast(p, 1.25j)) == 1.25j
-        assert complex(cast(p, complex(0,INF))) == complex(0,INF)
-        assert complex(cast(p, -INF)) == -INF
+        assert float(cast(p, INF*1j)) == INF*1j
+        assert float(cast(p, -INF)) == -INF
         if name == "float":
             assert complex(cast(p, 1.1j)) != 1.1j         # rounding error
             assert complex(cast(p, 1E200+3j)) == INF+3j   # limited range
-            assert complex(cast(p, complex(3,1E200))) == complex(3,INF) # limited range
+            assert complex(cast(p, 3+1E200j)) == 3+INF*1j # limited range
 
-        assert cast(p, -1.1j) == cast(p, -1.1j)
+        assert cast(p, -1.1j) != cast(p, -1.1j)
         assert repr(complex(cast(p, -0.0)).real) == '-0.0'
-        #assert repr(complex(cast(p, -0j))) == '-0j'   # http://bugs.python.org/issue29602
-        assert complex(cast(p, b'\x09')) == 9.0 + 0j
-        assert complex(cast(p, u+'\x09')) == 9.0 + 0j
-        assert complex(cast(p, True)) == 1.0 + 0j
+        assert repr(complex(cast(p, -0j))) == '-0j'
+        assert complex(cast(p, '\x09')) == 9.0
+        assert complex(cast(p, True)) == 1.0
         py.test.raises(TypeError, cast, p, None)
         #
-        py.test.raises(TypeError, cast, new_primitive_type(name), 1+0j)
-        #
-        for basetype in ["char", "int", "uint64_t", "float",
-                         "double", "long double"]:
-            baseobj = cast(new_primitive_type(basetype), 65)
-            py.test.raises(TypeError, complex, baseobj)
-        #
-        BArray = new_array_type(new_pointer_type(p), 10)
-        x = newp(BArray, None)
-        x[5] = 12.34 + 56.78j
-        assert type(x[5]) is complex
-        assert abs(x[5] - (12.34 + 56.78j)) < 1e-5
-        assert (x[5] == 12.34 + 56.78j) == (name == "double")  # rounding error
-        #
-        class Foo:
-            def __complex__(self):
-                return 2 + 3j
-        assert complex(Foo()) == 2 + 3j
-        assert complex(cast(p, Foo())) == 2 + 3j
-    py.test.raises(TypeError, cast, new_primitive_type("int"), 1+0j)
+        py.test.raises(cast, new_primitive_type(name), 1+2j)
+    py.test.raises(cast, new_primitive_type("int"), 1+2j)
 
 def test_character_type():
     p = new_primitive_type("char")
@@ -1124,34 +1105,6 @@
     BSShort = new_primitive_type("short")
     assert f(3, cast(BSChar, -3), cast(BUChar, 200), cast(BSShort, -5)) == 192
 
-def test_call_function_24():
-    BFloat = new_primitive_type("float")
-    BFloatComplex = new_primitive_type("float _Complex")
-    BFunc3 = new_function_type((BFloat, BFloat), BFloatComplex, False)
-    if 0:   # libffi returning nonsense silently, so logic disabled for now
-        f = cast(BFunc3, _testfunc(24))
-        result = f(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-    else:
-        f = cast(BFunc3, _testfunc(9))
-        py.test.raises(NotImplementedError, f, 12.3, 34.5)
-
-def test_call_function_25():
-    BDouble = new_primitive_type("double")
-    BDoubleComplex = new_primitive_type("double _Complex")
-    BFunc3 = new_function_type((BDouble, BDouble), BDoubleComplex, False)
-    if 0:   # libffi returning nonsense silently, so logic disabled for now
-        f = cast(BFunc3, _testfunc(25))
-        result = f(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-10) # inexact
-    else:
-        f = cast(BFunc3, _testfunc(9))
-        py.test.raises(NotImplementedError, f, 12.3, 34.5)
-
 def test_cannot_call_with_a_autocompleted_struct():
     BSChar = new_primitive_type("signed char")
     BDouble = new_primitive_type("double")
@@ -3843,7 +3796,7 @@
 
 def test_char_pointer_conversion():
     import warnings
-    assert __version__.startswith(("1.8", "1.9", "1.10", "1.11")), (
+    assert __version__.startswith(("1.8", "1.9", "1.10")), (
         "consider turning the warning into an error")
     BCharP = new_pointer_type(new_primitive_type("char"))
     BIntP = new_pointer_type(new_primitive_type("int"))
diff --git a/pypy/module/_cffi_backend/test/test_parse_c_type.py b/pypy/module/_cffi_backend/test/test_parse_c_type.py
--- a/pypy/module/_cffi_backend/test/test_parse_c_type.py
+++ b/pypy/module/_cffi_backend/test/test_parse_c_type.py
@@ -148,8 +148,6 @@
             ("long int", cffi_opcode.PRIM_LONG),
             ("unsigned short", cffi_opcode.PRIM_USHORT),
             ("long double", cffi_opcode.PRIM_LONGDOUBLE),
-            (" float  _Complex", cffi_opcode.PRIM_FLOATCOMPLEX),
-            ("double _Complex ", cffi_opcode.PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -275,11 +273,6 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
-    #
-    parse_error("_Complex", "identifier expected", 0)
-    parse_error("int _Complex", "_Complex type combination unsupported", 4)
-    parse_error("long double _Complex", "_Complex type combination unsupported",
-                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -1819,68 +1819,6 @@
         assert lib.f.__get__(42) is lib.f
         assert lib.f.__get__(42, int) is lib.f
 
-    def test_function_returns_float_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "float _Complex f1(float a, float b);",
-            "test_function_returns_float_complex", """
-            #include <complex.h>
-            static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-        """, min_version=(1, 11, 0))
-        result = lib.f1(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-
-    def test_function_returns_double_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "double _Complex f1(double a, double b);",
-            "test_function_returns_double_complex", """
-            #include <complex.h>
-            static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-        """, min_version=(1, 11, 0))
-        result = lib.f1(1.25, 5.1)
-        assert type(result) == complex
-        assert result.real == 1.25   # exact
-        assert result.imag == 2*5.1  # exact
-
-    def test_function_argument_float_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "float f1(float _Complex x);",
-            "test_function_argument_float_complex", """
-            #include <complex.h>
-            static float f1(float _Complex x) { return cabsf(x); }
-        """, min_version=(1, 11, 0))
-        x = complex(12.34, 56.78)
-        result = lib.f1(x)
-        assert abs(result - abs(x)) < 1e-5
-        result2 = lib.f1(ffi.cast("float _Complex", x))
-        assert result2 == result
-
-    def test_function_argument_double_complex(self):
-        import sys
-        if sys.platform == 'win32':
-            skip("MSVC may not support _Complex")
-        ffi, lib = self.prepare(
-            "double f1(double _Complex);",
-            "test_function_argument_double_complex", """
-            #include <complex.h>
-            static double f1(double _Complex x) { return cabs(x); }
-        """, min_version=(1, 11, 0))
-        x = complex(12.34, 56.78)
-        result = lib.f1(x)
-        assert abs(result - abs(x)) < 1e-11
-        result2 = lib.f1(ffi.cast("double _Complex", x))
-        assert result2 == result
-
     def test_typedef_array_dotdotdot(self):
         ffi, lib = self.prepare("""
             typedef int foo_t[...], bar_t[...];
diff --git a/pypy/module/_cffi_backend/wrapper.py b/pypy/module/_cffi_backend/wrapper.py
--- a/pypy/module/_cffi_backend/wrapper.py
+++ b/pypy/module/_cffi_backend/wrapper.py
@@ -8,7 +8,6 @@
 from pypy.module._cffi_backend.cdataobj import W_CData
 from pypy.module._cffi_backend.cdataobj import W_CDataPtrToStructOrUnion
 from pypy.module._cffi_backend.ctypeptr import W_CTypePtrOrArray
-from pypy.module._cffi_backend.ctypeptr import W_CTypePointer
 from pypy.module._cffi_backend.ctypefunc import W_CTypeFunc
 from pypy.module._cffi_backend.ctypestruct import W_CTypeStructOrUnion
 from pypy.module._cffi_backend import allocator
@@ -84,9 +83,8 @@
                 #
                 ctype._call(self.fnptr, args_w)    # returns w_None
                 #
-                ctyperesptr = w_result_cdata.ctype
-                assert isinstance(ctyperesptr, W_CTypePointer)
-                return w_result_cdata._do_getitem(ctyperesptr, 0)
+                assert isinstance(w_result_cdata, W_CDataPtrToStructOrUnion)
+                return w_result_cdata.structobj
             else:
                 args_w = args_w[:]
                 prepare_args(space, rawfunctype, args_w, 0)
@@ -111,14 +109,13 @@
 @jit.unroll_safe
 def prepare_args(space, rawfunctype, args_w, start_index):
     # replaces struct/union arguments with ptr-to-struct/union arguments
-    # as well as complex numbers
     locs = rawfunctype.nostruct_locs
     fargs = rawfunctype.nostruct_ctype.fargs
     for i in range(start_index, len(locs)):
         if locs[i] != 'A':
             continue
         w_arg = args_w[i]
-        farg = fargs[i]      # <ptr to struct/union/complex>
+        farg = fargs[i]      # <ptr to struct/union>
         assert isinstance(farg, W_CTypePtrOrArray)
         if isinstance(w_arg, W_CData) and w_arg.ctype is farg.ctitem:
             # fast way: we are given a W_CData "struct", so just make
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,27 +229,13 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == (
-           "<cdef source string>:1: foo: a function with only '(...)' "
-           "as argument is not correct C")
+    assert str(e.value) == \
+           "foo: a function with only '(...)' as argument is not correct C"
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert str(e.value).startswith(
-        'cannot parse "x y z"\n<cdef source string>:1:')
-    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
-    assert str(e.value).startswith(
-        'cannot parse "x y z"\n<cdef source string>:4:')
-
-def test_error_custom_lineno():
-    ffi = FFI()
-    e = py.test.raises(CDefError, ffi.cdef, """
-# 42 "foobar"
-
-    a b c d
-    """)
-    assert str(e.value).startswith('parse error\nfoobar:43:')
+    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -293,8 +279,7 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
-                            " unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -452,9 +437,3 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
-
-def test_error_invalid_syntax_for_cdef():
-    ffi = FFI()
-    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
-    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
-                            'this construct is valid C but not valid in cdef()')
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -240,18 +240,15 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
-        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert X == (typename in ('float _Complex', 'double _Complex'))
-        assert I + F + C + X == 1      # one and only one of them is true
+        assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1705,8 +1705,6 @@
             "ptrdiff_t",
             "size_t",
             "ssize_t",
-            'float _Complex',
-            'double _Complex',
             ])
         for name in PRIMITIVE_TO_INDEX:
             x = ffi.sizeof(name)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_parse_c_type.py
@@ -156,8 +156,6 @@
             ("long int", lib._CFFI_PRIM_LONG),
             ("unsigned short", lib._CFFI_PRIM_USHORT),
             ("long double", lib._CFFI_PRIM_LONGDOUBLE),
-            (" float  _Complex", lib._CFFI_PRIM_FLOATCOMPLEX),
-            ("double _Complex ", lib._CFFI_PRIM_DOUBLECOMPLEX),
             ]:
         assert parse(simple_type) == ['->', Prim(expected)]
 
@@ -283,11 +281,6 @@
     parse_error("int[5](*)", "unexpected symbol", 6)
     parse_error("int a(*)", "identifier expected", 6)
     parse_error("int[123456789012345678901234567890]", "number too large", 4)
-    #
-    parse_error("_Complex", "identifier expected", 0)
-    parse_error("int _Complex", "_Complex type combination unsupported", 4)
-    parse_error("long double _Complex", "_Complex type combination unsupported",
-                12)
 
 def test_number_too_large():
     num_max = sys.maxsize
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_realize_c_type.py
@@ -48,6 +48,7 @@
     for name in cffi_opcode.PRIMITIVE_TO_INDEX:
         check(name, name)
 
+
 def check_func(input, expected_output=None):
     import _cffi_backend
     ffi = _cffi_backend.FFI()
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -1553,8 +1553,7 @@
         res = lib.bar(4, 5)
     assert res == 0
     assert f.getvalue() == (
-        b"extern \"Python\": function _CFFI_test_extern_python_1.bar() called, "
-        b"but no code was attached "
+        b"extern \"Python\": function bar() called, but no code was attached "
         b"to it yet with @ffi.def_extern().  Returning 0.\n")
 
     @ffi.def_extern("bar")
@@ -2002,60 +2001,6 @@
     """)
     assert lib.f1(52).a == 52
 
-def test_function_returns_float_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("float _Complex f1(float a, float b);");
-    lib = verify(ffi, "test_function_returns_float_complex", """
-        #include <complex.h>
-        static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
-    result = lib.f1(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert (result.imag != 2*5.1) and (abs(result.imag - 2*5.1) < 1e-5) # inexact
-
-def test_function_returns_double_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("double _Complex f1(double a, double b);");
-    lib = verify(ffi, "test_function_returns_double_complex", """
-        #include <complex.h>
-        static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
-    result = lib.f1(1.25, 5.1)
-    assert type(result) == complex
-    assert result.real == 1.25   # exact
-    assert result.imag == 2*5.1  # exact
-
-def test_function_argument_float_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("float f1(float _Complex x);");
-    lib = verify(ffi, "test_function_argument_float_complex", """
-        #include <complex.h>
-        static float f1(float _Complex x) { return cabsf(x); }
-    """)
-    x = complex(12.34, 56.78)
-    result = lib.f1(x)
-    assert abs(result - abs(x)) < 1e-5
-
-def test_function_argument_double_complex():
-    if sys.platform == 'win32':
-        py.test.skip("MSVC may not support _Complex")
-    ffi = FFI()
-    ffi.cdef("double f1(double _Complex);");
-    lib = verify(ffi, "test_function_argument_double_complex", """
-        #include <complex.h>
-        static double f1(double _Complex x) { return cabs(x); }
-    """)
-    x = complex(12.34, 56.78)
-    result = lib.f1(x)
-    assert abs(result - abs(x)) < 1e-11
-
 def test_typedef_array_dotdotdot():
     ffi = FFI()
     ffi.cdef("""
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -220,18 +220,15 @@
         tp = model.PrimitiveType(typename)
         C = tp.is_char_type()
         F = tp.is_float_type()
-        X = tp.is_complex_type()
         I = tp.is_integer_type()
         assert C == (typename in ('char', 'wchar_t'))
         assert F == (typename in ('float', 'double', 'long double'))
-        assert X == (typename in ('float _Complex', 'double _Complex'))
-        assert I + F + C + X == 1      # one and only one of them is true
+        assert I + F + C == 1      # one and only one of them is true
 
 def test_all_integer_and_float_types():
     typenames = []
     for typename in all_primitive_types:
         if (all_primitive_types[typename] == 'c' or
-            all_primitive_types[typename] == 'j' or    # complex
             typename == '_Bool' or typename == 'long double'):
             pass
         else:

From pypy.commits at gmail.com  Fri Jun  2 06:51:52 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:52 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: merge cffi 1.10.1 (from
 relevant parts of "hg -R cffi diff --rev 2919 --rev 2933")
Message-ID: <59314348.c486df0a.b82b1.d66c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91492:3a5ee644c6e4
Date: 2017-06-02 13:40 +0300
http://bitbucket.org/pypy/pypy/changeset/3a5ee644c6e4/

Log:	merge cffi 1.10.1 (from relevant parts of "hg -R cffi diff --rev
	2919 --rev 2933") (grafted from
	e9ef3455dec54ac63ab2cfd21ccb211fe74f7868)

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -8,7 +8,7 @@
    the same works for the other two macros.  Py_DEBUG implies them,
    but not the other way around.
 */
-#ifndef _CFFI_USE_EMBEDDING
+#if !defined(_CFFI_USE_EMBEDDING) && !defined(Py_LIMITED_API)
 #  include <pyconfig.h>
 #  if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_REF_DEBUG)
 #    define Py_LIMITED_API
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -16,6 +16,7 @@
 except ImportError:
     lock = None
 
+CDEF_SOURCE_STRING = "<cdef source string>"
 _r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$",
                         re.DOTALL | re.MULTILINE)
 _r_define  = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)"
@@ -258,15 +259,21 @@
                 ctn.discard(name)
         typenames += sorted(ctn)
         #
-        csourcelines = ['typedef int %s;' % typename for typename in typenames]
+        csourcelines = []
+        csourcelines.append('# 1 "<cdef automatic initialization code>"')
+        for typename in typenames:
+            csourcelines.append('typedef int %s;' % typename)
         csourcelines.append('typedef int __dotdotdotint__, __dotdotdotfloat__,'
                             ' __dotdotdot__;')
+        # this forces pycparser to consider the following in the file
+        # called <cdef source string> from line 1
+        csourcelines.append('# 1 "%s"' % (CDEF_SOURCE_STRING,))
         csourcelines.append(csource)
-        csource = '\n'.join(csourcelines)
+        fullcsource = '\n'.join(csourcelines)
         if lock is not None:
             lock.acquire()     # pycparser is not thread-safe...
         try:
-            ast = _get_parser().parse(csource)
+            ast = _get_parser().parse(fullcsource)
         except pycparser.c_parser.ParseError as e:
             self.convert_pycparser_error(e, csource)
         finally:
@@ -276,17 +283,17 @@
         return ast, macros, csource
 
     def _convert_pycparser_error(self, e, csource):
-        # xxx look for ":NUM:" at the start of str(e) and try to interpret
-        # it as a line number
+        # xxx look for "<cdef source string>:NUM:" at the start of str(e)
+        # and interpret that as a line number.  This will not work if
+        # the user gives explicit ``# NUM "FILE"`` directives.
         line = None
         msg = str(e)
-        if msg.startswith(':') and ':' in msg[1:]:
-            linenum = msg[1:msg.find(':',1)]
-            if linenum.isdigit():
-                linenum = int(linenum, 10)
-                csourcelines = csource.splitlines()
-                if 1 <= linenum <= len(csourcelines):
-                    line = csourcelines[linenum-1]
+        match = re.match(r"%s:(\d+):" % (CDEF_SOURCE_STRING,), msg)
+        if match:
+            linenum = int(match.group(1), 10)
+            csourcelines = csource.splitlines()
+            if 1 <= linenum <= len(csourcelines):
+                line = csourcelines[linenum-1]
         return line
 
     def convert_pycparser_error(self, e, csource):
@@ -321,10 +328,12 @@
                 break
         else:
             assert 0
+        current_decl = None
         #
         try:
             self._inside_extern_python = '__cffi_extern_python_stop'
             for decl in iterator:
+                current_decl = decl
                 if isinstance(decl, pycparser.c_ast.Decl):
                     self._parse_decl(decl)
                 elif isinstance(decl, pycparser.c_ast.Typedef):
@@ -348,7 +357,13 @@
                 elif decl.__class__.__name__ == 'Pragma':
                     pass    # skip pragma, only in pycparser 2.15
                 else:
-                    raise CDefError("unrecognized construct", decl)
+                    raise CDefError("unexpected <%s>: this construct is valid "
+                                    "C but not valid in cdef()" %
+                                    decl.__class__.__name__, decl)
+        except CDefError as e:
+            if len(e.args) == 1:
+                e.args = e.args + (current_decl,)
+            raise
         except FFIError as e:
             msg = self._convert_pycparser_error(e, csource)
             if msg:
diff --git a/lib_pypy/cffi/error.py b/lib_pypy/cffi/error.py
--- a/lib_pypy/cffi/error.py
+++ b/lib_pypy/cffi/error.py
@@ -5,10 +5,13 @@
 class CDefError(Exception):
     def __str__(self):
         try:
-            line = 'line %d: ' % (self.args[1].coord.line,)
+            current_decl = self.args[1]
+            filename = current_decl.coord.file
+            linenum = current_decl.coord.line
+            prefix = '%s:%d: ' % (filename, linenum)
         except (AttributeError, TypeError, IndexError):
-            line = ''
-        return '%s%s' % (line, self.args[0])
+            prefix = ''
+        return '%s%s' % (prefix, self.args[0])
 
 class VerificationError(Exception):
     """ An error raised when verification fails
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -6,6 +6,7 @@
                       'extra_objects', 'depends']
 
 def get_extension(srcfilename, modname, sources=(), **kwds):
+    _hack_at_distutils()
     from distutils.core import Extension
     allsources = [srcfilename]
     for src in sources:
@@ -15,6 +16,7 @@
 def compile(tmpdir, ext, compiler_verbose=0, debug=None):
     """Compile a C extension module using distutils."""
 
+    _hack_at_distutils()
     saved_environ = os.environ.copy()
     try:
         outputfilename = _build(tmpdir, ext, compiler_verbose, debug)
@@ -113,3 +115,13 @@
     f = cStringIO.StringIO()
     _flatten(x, f)
     return f.getvalue()
+
+def _hack_at_distutils():
+    # Windows-only workaround for some configurations: see
+    # https://bugs.python.org/issue23246 (Python 2.7 with 
+    # a specific MS compiler suite download)
+    if sys.platform == "win32":
+        try:
+            import setuptools    # for side-effects, patches distutils
+        except ImportError:
+            pass
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -1479,6 +1479,12 @@
                     _patch_for_embedding(patchlist)
                 if target != '*':
                     _patch_for_target(patchlist, target)
+                if compiler_verbose:
+                    if tmpdir == '.':
+                        msg = 'the current directory is'
+                    else:
+                        msg = 'setting the current directory to'
+                    print('%s %r' % (msg, os.path.abspath(tmpdir)))
                 os.chdir(tmpdir)
                 outputfilename = ffiplatform.compile('.', ext,
                                                      compiler_verbose, debug)
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -26,16 +26,6 @@
                 s = s.encode('ascii')
             super(NativeIO, self).write(s)
 
-def _hack_at_distutils():
-    # Windows-only workaround for some configurations: see
-    # https://bugs.python.org/issue23246 (Python 2.7 with 
-    # a specific MS compiler suite download)
-    if sys.platform == "win32":
-        try:
-            import setuptools    # for side-effects, patches distutils
-        except ImportError:
-            pass
-
 
 class Verifier(object):
 
@@ -126,7 +116,7 @@
         return basename
 
     def get_extension(self):
-        _hack_at_distutils() # backward compatibility hack
+        ffiplatform._hack_at_distutils() # backward compatibility hack
         if not self._has_source:
             with self.ffi._lock:
                 if not self._has_source:

From pypy.commits at gmail.com  Fri Jun  2 06:51:54 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 02 Jun 2017 03:51:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: document cffi 1.10.1, PGO
Message-ID: <5931434a.9199df0a.fedc3.db97@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91493:0961eb1c2589
Date: 2017-06-02 13:49 +0300
http://bitbucket.org/pypy/pypy/changeset/0961eb1c2589/

Log:	document cffi 1.10.1, PGO

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,10 +10,9 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-We continue to make incremental improvements to our C-API
-compatibility layer (cpyext). PyPy2 can now import and run many C-extension
-packages, among the most notable are Numpy, Cython, and Pandas. Performance may
-be slower than CPython, especially for frequently-called short C functions.
+This release enables `profile guided optimization` of the base interpreter,
+which may make unjitted code run faster.
+
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
@@ -24,8 +23,8 @@
 We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
-CFFI_ has been updated to 1.11, improving an already great package for
-interfacing with C.
+CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.10.1,
+improving an already great package for interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -43,6 +42,7 @@
 improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
 with making RPython's JIT even better.
 
+.. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html

From pypy.commits at gmail.com  Fri Jun  2 09:45:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 06:45:00 -0700 (PDT)
Subject: [pypy-commit] cffi default: Mention that 0.10.1 is only inside PyPy
 5.8.0
Message-ID: <59316bdc.05421c0a.4cd7a.2214@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2968:f4ff36763712
Date: 2017-06-02 15:44 +0200
http://bitbucket.org/cffi/cffi/changeset/f4ff36763712/

Log:	Mention that 0.10.1 is only inside PyPy 5.8.0

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -22,6 +22,8 @@
 v1.10.1
 =======
 
+(only released inside PyPy 5.8.0)
+
 * Fixed the line numbers reported in case of ``cdef()`` errors.
   Also, I just noticed, but pycparser always supported the preprocessor
   directive ``# 42 "foo.h"`` to mean "from the next line, we're in file

From pypy.commits at gmail.com  Fri Jun  2 09:45:28 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 06:45:28 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: I think this test needs to
 change too (from 'hg diff -r 486d919c0b87 -r
Message-ID: <59316bf8.089a1c0a.b027.b635@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r91494:0ec86a2b43ab
Date: 2017-06-02 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/0ec86a2b43ab/

Log:	I think this test needs to change too (from 'hg diff -r 486d919c0b87
	-r 93fa52b7eed3 testing/' inside cffi)

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')

From pypy.commits at gmail.com  Fri Jun  2 09:45:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 06:45:34 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: Give the version 1.10.1 to
 cffi
Message-ID: <59316bfe.cf921c0a.6a757.1773@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.5-5.x
Changeset: r91497:c29e21f6dd75
Date: 2017-06-02 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/c29e21f6dd75/

Log:	Give the version 1.10.1 to cffi (grafted from
	1bfb11c86cfd81a86d54823f85aad0ca5d89fd98)

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.10.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.10.1"
+__version_info__ = (1, 10, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.10.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.10.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Fri Jun  2 09:45:30 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 06:45:30 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: I think this test needs to
 change too (from 'hg diff -r 486d919c0b87 -r
Message-ID: <59316bfa.09631c0a.77beb.c9ab@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.5-5.x
Changeset: r91495:1948ff94a48d
Date: 2017-06-02 15:30 +0200
http://bitbucket.org/pypy/pypy/changeset/1948ff94a48d/

Log:	I think this test needs to change too (from 'hg diff -r 486d919c0b87
	-r 93fa52b7eed3 testing/' inside cffi) (grafted from
	0ec86a2b43ab82ef038fdad17d8b85eb563f1dd8)

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -229,13 +229,27 @@
     # this checks that we get a sensible error if we try "int foo(...);"
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "int foo(...);")
-    assert str(e.value) == \
-           "foo: a function with only '(...)' as argument is not correct C"
+    assert str(e.value) == (
+           "<cdef source string>:1: foo: a function with only '(...)' "
+           "as argument is not correct C")
 
 def test_parse_error():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, " x y z ")
-    assert re.match(r'cannot parse "x y z"\n:\d+:', str(e.value))
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:1:')
+    e = py.test.raises(CDefError, ffi.cdef, "\n\n\n x y z ")
+    assert str(e.value).startswith(
+        'cannot parse "x y z"\n<cdef source string>:4:')
+
+def test_error_custom_lineno():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, """
+# 42 "foobar"
+
+    a b c d
+    """)
+    assert str(e.value).startswith('parse error\nfoobar:43:')
 
 def test_cannot_declare_enum_later():
     ffi = FFI()
@@ -279,7 +293,8 @@
 def test_unknown_argument_type():
     ffi = FFI()
     e = py.test.raises(CDefError, ffi.cdef, "void f(foobarbazzz);")
-    assert str(e.value) == ("f arg 1: unknown type 'foobarbazzz' (if you meant"
+    assert str(e.value) == ("<cdef source string>:1: f arg 1:"
+                            " unknown type 'foobarbazzz' (if you meant"
                             " to use the old C syntax of giving untyped"
                             " arguments, it is not supported)")
 
@@ -437,3 +452,9 @@
             ffi._parser._declarations['extern_python foobar'] !=
             ffi._parser._declarations['function bok'] ==
             ffi._parser._declarations['extern_python bzrrr'])
+
+def test_error_invalid_syntax_for_cdef():
+    ffi = FFI()
+    e = py.test.raises(CDefError, ffi.cdef, 'void foo(void) {}')
+    assert str(e.value) == ('<cdef source string>:1: unexpected <FuncDef>: '
+                            'this construct is valid C but not valid in cdef()')

From pypy.commits at gmail.com  Fri Jun  2 09:45:32 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 06:45:32 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Give the version 1.10.1 to
 cffi
Message-ID: <59316bfc.05421c0a.4cd7a.2240@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r91496:1bfb11c86cfd
Date: 2017-06-02 15:40 +0200
http://bitbucket.org/pypy/pypy/changeset/1bfb11c86cfd/

Log:	Give the version 1.10.1 to cffi

diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO
--- a/lib_pypy/cffi.egg-info/PKG-INFO
+++ b/lib_pypy/cffi.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: cffi
-Version: 1.10.0
+Version: 1.10.1
 Summary: Foreign Function Interface for Python calling C code.
 Home-page: http://cffi.readthedocs.org
 Author: Armin Rigo, Maciej Fijalkowski
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,8 +4,8 @@
 from .api import FFI
 from .error import CDefError, FFIError, VerificationError, VerificationMissing
 
-__version__ = "1.10.0"
-__version_info__ = (1, 10, 0)
+__version__ = "1.10.1"
+__version_info__ = (1, 10, 1)
 
 # The verifier module file names are based on the CRC32 of a string that
 # contains the following version number.  It may be older than __version__
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib import rdynload, clibffi
 from rpython.rtyper.lltypesystem import rffi
 
-VERSION = "1.10.0"
+VERSION = "1.10.1"
 
 FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI
 try:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1,7 +1,7 @@
 # ____________________________________________________________
 
 import sys
-assert __version__ == "1.10.0", ("This test_c.py file is for testing a version"
+assert __version__ == "1.10.1", ("This test_c.py file is for testing a version"
                                  " of cffi that differs from the one that we"
                                  " get from 'import _cffi_backend'")
 if sys.version_info < (3,):

From pypy.commits at gmail.com  Fri Jun  2 11:30:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 08:30:51 -0700 (PDT)
Subject: [pypy-commit] cffi default: an extra test
Message-ID: <593184ab.5888df0a.9e851.70db@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2969:fd8ab01804fb
Date: 2017-06-02 17:30 +0200
http://bitbucket.org/cffi/cffi/changeset/fd8ab01804fb/

Log:	an extra test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2246,6 +2246,22 @@
         x = cast(BWChar, -1)
         py.test.raises(ValueError, string, x)
 
+def test_wchar_variants_mix():
+    BWChar  = new_primitive_type("wchar_t")
+    BChar16 = new_primitive_type("char16_t")
+    BChar32 = new_primitive_type("char32_t")
+    assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+    assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+    #
+    BChar16A = new_array_type(new_pointer_type(BChar16), None)
+    BChar32A = new_array_type(new_pointer_type(BChar32), None)
+    x = cast(BChar32, 'A')
+    py.test.raises(TypeError, newp, BChar16A, [x])
+    x = cast(BChar16, 'A')
+    py.test.raises(TypeError, newp, BChar32A, [x])
+
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the

From pypy.commits at gmail.com  Fri Jun  2 12:25:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 02 Jun 2017 09:25:37 -0700 (PDT)
Subject: [pypy-commit] cffi default: More tests for char16_t and char32_t
Message-ID: <59319181.cc2d1c0a.a72d4.9276@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2970:6c465c147687
Date: 2017-06-02 18:25 +0200
http://bitbucket.org/cffi/cffi/changeset/6c465c147687/

Log:	More tests for char16_t and char32_t

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -1936,7 +1936,11 @@
         assert string(a, 8).startswith(b'ABC')  # may contain additional garbage
 
 def test_string_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     assert string(cast(BWChar, 42)) == u+'*'
     assert string(cast(BWChar, 0x4253)) == u+'\u4253'
     assert string(cast(BWChar, 0)) == u+'\x00'
@@ -3488,14 +3492,15 @@
     py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
     py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
     #
-    BUniChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BUniChar), None)
-    p = newp(BArray, u+"foobar")
-    p[2:5] = [u+"*", u+"Z", u+"T"]
-    p[1:3] = u+"XY"
-    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
-    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
-    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BUniChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BUniChar), None)
+        p = newp(BArray, u+"foobar")
+        p[2:5] = [u+"*", u+"Z", u+"T"]
+        p[1:3] = u+"XY"
+        assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+        py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+        py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
 
 def test_void_p_arithmetic():
     BVoid = new_void_type()
@@ -3808,10 +3813,12 @@
     p0 = p
     assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
     assert unpack(p+1, 5) == b"bc\x00de"
-    BWChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
-    p = newp(BArray, u"abc\x00def")
-    assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BWChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
+        p = newp(BArray, u"abc\x00def")
+        assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
 
     for typename, samples in [
             ("uint8_t",  [0, 2**8-1]),

From pypy.commits at gmail.com  Fri Jun  2 13:00:13 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 02 Jun 2017 10:00:13 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Let OrderedDict.__init__ behave like
 CPython wrt. subclasses overridding __setitem__
Message-ID: <5931999d.89341c0a.8c38.4a8b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91498:c13ae2a7e07a
Date: 2017-06-02 18:00 +0100
http://bitbucket.org/pypy/pypy/changeset/c13ae2a7e07a/

Log:	Let OrderedDict.__init__ behave like CPython wrt. subclasses
	overridding __setitem__

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -15,6 +15,21 @@
     cases but is nonsensical in other cases.  This is officially
     forbidden by the CPython docs, so we forbid it explicitly for now.
     '''
+    def __init__(*args, **kwds):
+        '''Initialize an ordered dictionary.  The signature is the same as
+        regular dictionaries, but keyword arguments are not recommended because
+        their insertion order is arbitrary.
+
+        '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'OrderedDict' object "
+                            "needs an argument")
+        self, *args = args
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        self.__update(*args, **kwds)
+
+    update = __update = _collections_abc.MutableMapping.update
 
     def __reversed__(self):
         return reversed_dict(self)
diff --git a/pypy/module/_collections/test/test_ordereddict.py b/pypy/module/_collections/test/test_ordereddict.py
--- a/pypy/module/_collections/test/test_ordereddict.py
+++ b/pypy/module/_collections/test/test_ordereddict.py
@@ -12,3 +12,13 @@
         d = OrderedDict()
         d[1] = d
         assert repr(d) == 'OrderedDict([(1, ...)])'
+
+    def test_subclass(self):
+        from _collections import OrderedDict
+        class MyODict(OrderedDict):
+            def __setitem__(self, key, value):
+                super().__setitem__(key, 42)
+        d = MyODict(x=1)
+        assert d['x'] == 42
+        d.update({'y': 2})
+        assert d['y'] == 42

From pypy.commits at gmail.com  Sat Jun  3 14:35:56 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 03 Jun 2017 11:35:56 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: Let OrderedDict.__init__
 behave like CPython wrt. subclasses overridding __setitem__
Message-ID: <5933018c.11addf0a.2b75b.eb4a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91499:9d7a5438be1d
Date: 2017-06-03 21:31 +0300
http://bitbucket.org/pypy/pypy/changeset/9d7a5438be1d/

Log:	Let OrderedDict.__init__ behave like CPython wrt. subclasses
	overridding __setitem__ (grafted from
	c13ae2a7e07aed0d1756a73bbfc9c389057c35e1)

diff --git a/lib_pypy/_pypy_collections.py b/lib_pypy/_pypy_collections.py
--- a/lib_pypy/_pypy_collections.py
+++ b/lib_pypy/_pypy_collections.py
@@ -15,6 +15,21 @@
     cases but is nonsensical in other cases.  This is officially
     forbidden by the CPython docs, so we forbid it explicitly for now.
     '''
+    def __init__(*args, **kwds):
+        '''Initialize an ordered dictionary.  The signature is the same as
+        regular dictionaries, but keyword arguments are not recommended because
+        their insertion order is arbitrary.
+
+        '''
+        if not args:
+            raise TypeError("descriptor '__init__' of 'OrderedDict' object "
+                            "needs an argument")
+        self, *args = args
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        self.__update(*args, **kwds)
+
+    update = __update = _collections_abc.MutableMapping.update
 
     def __reversed__(self):
         return reversed_dict(self)
diff --git a/pypy/module/_collections/test/test_ordereddict.py b/pypy/module/_collections/test/test_ordereddict.py
--- a/pypy/module/_collections/test/test_ordereddict.py
+++ b/pypy/module/_collections/test/test_ordereddict.py
@@ -12,3 +12,13 @@
         d = OrderedDict()
         d[1] = d
         assert repr(d) == 'OrderedDict([(1, ...)])'
+
+    def test_subclass(self):
+        from _collections import OrderedDict
+        class MyODict(OrderedDict):
+            def __setitem__(self, key, value):
+                super().__setitem__(key, 42)
+        d = MyODict(x=1)
+        assert d['x'] == 42
+        d.update({'y': 2})
+        assert d['y'] == 42

From pypy.commits at gmail.com  Sat Jun  3 14:35:58 2017
From: pypy.commits at gmail.com (mattip)
Date: Sat, 03 Jun 2017 11:35:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: document graft of c13ae2a7e07a to
 release 3.5
Message-ID: <5933018e.92141c0a.33317.b578@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91500:fd9441d99cc0
Date: 2017-06-03 21:35 +0300
http://bitbucket.org/pypy/pypy/changeset/fd9441d99cc0/

Log:	document graft of c13ae2a7e07a to release 3.5

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -175,6 +175,8 @@
   * Get closer to supporting 32 bit windows, translation now succeeds and most
     lib-python/3/test runs
   * Call ``sys.__interactivehook__`` at startup
+  * Let ``OrderedDict.__init__`` behave like CPython wrt. subclasses
+    overridding ``__setitem__``
 
 * Performance improvements:
 

From pypy.commits at gmail.com  Sat Jun  3 20:08:22 2017
From: pypy.commits at gmail.com (stefanor)
Date: Sat, 03 Jun 2017 17:08:22 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: pypy3 manpage
Message-ID: <59334f76.4c3a1c0a.f0b46.7e81@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: py3.5
Changeset: r91501:e00caafaf849
Date: 2017-06-03 17:07 -0700
http://bitbucket.org/pypy/pypy/changeset/e00caafaf849/

Log:	pypy3 manpage

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -222,8 +222,8 @@
 # -- Options for manpage output-------------------------------------------------
 
 man_pages = [
-  ('man/pypy.1', 'pypy',
-   u'fast, compliant alternative implementation of the Python language',
+  ('man/pypy3.1', 'pypy3',
+   u'fast, compliant alternative implementation of the Python 3 language',
    u'The PyPy Project', 1)
 ]
 
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy3.1.rst
copy from pypy/doc/man/pypy.1.rst
copy to pypy/doc/man/pypy3.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy3.1.rst
@@ -1,14 +1,14 @@
-======
- pypy
-======
+=======
+ pypy3
+=======
 
-.. note: this is turned into a regular man page "pypy.1" by
+.. note: this is turned into a regular man page "pypy3.1" by
    doing "make man" in pypy/doc/
 
 SYNOPSIS
 ========
 
-``pypy`` [*options*]
+``pypy3`` [*options*]
 [``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
 [*arg*\ ...]
 
@@ -69,7 +69,7 @@
 ===========
 
 ``PYTHONPATH``
-    Add directories to pypy's module search path.
+    Add directories to pypy3's module search path.
     The format is the same as shell's ``PATH``.
 
 ``PYTHONSTARTUP``
@@ -132,4 +132,4 @@
 SEE ALSO
 ========
 
-**python**\ (1)
+**python3**\ (1)

From pypy.commits at gmail.com  Sat Jun  3 20:26:52 2017
From: pypy.commits at gmail.com (stefanor)
Date: Sat, 03 Jun 2017 17:26:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Expose SOABI in sysconfig in pypy3 too
Message-ID: <593353cc.4c3a1c0a.f0b46.8606@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: py3.5
Changeset: r91502:228f8b4206ae
Date: 2017-06-03 17:26 -0700
http://bitbucket.org/pypy/pypy/changeset/228f8b4206ae/

Log:	Expose SOABI in sysconfig in pypy3 too

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -5,5 +5,6 @@
 build_time_vars = {
     "EXT_SUFFIX": so_ext,
     "SHLIB_SUFFIX": so_ext,
+    "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
     "SO": so_ext  # deprecated in Python 3, for backward compatibility
 }

From pypy.commits at gmail.com  Sun Jun  4 01:03:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 03 Jun 2017 22:03:33 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: in-progress
Message-ID: <593394a5.83b2df0a.2176a.3206@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91503:d6d714960021
Date: 2017-06-04 07:01 +0200
http://bitbucket.org/pypy/pypy/changeset/d6d714960021/

Log:	in-progress

diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -107,8 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +133,12 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -42,6 +42,7 @@
     def cast_unicode(self, w_ob):
         space = self.space
         s = space.unicode_w(w_ob)
+        XXXXXXXXXXXXXX
         if len(s) != 1:
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
@@ -149,15 +150,15 @@
 
 
 class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
-    _attrs_            = ['is_signed']
-    _immutable_fields_ = ['is_signed']
+    _attrs_            = ['is_signed_wchar']
+    _immutable_fields_ = ['is_signed_wchar']
 
     _wchar_is_signed = rfficache.signof_c_type('wchar_t')
 
     def __init__(self, space, size, name, name_position, align):
-        W_CTypePrimitiveUniChar.__init__(self, space, size, name,
-                                         name_position, align)
-        self.is_signed = self._wchar_is_signed and (name == "wchar_t")
+        W_CTypePrimitiveCharOrUniChar.__init__(self, space, size, name,
+                                               name_position, align)
+        self.is_signed_wchar = self._wchar_is_signed and (name == "wchar_t")
         # "char16_t" and "char32_t" are always unsigned
 
     def cast_to_int(self, cdata):
@@ -185,32 +186,41 @@
             w_res = self.convert_to_object(ptr)
         return w_res
 
-    def _convert_to_charN_t(self, w_ob, size):
-        # returns a r_uint.  If size == 2, it is smaller than 0x10000
+    def _convert_to_charN_t(self, w_ob):
+        # returns a r_uint.  If self.size == 2, it is smaller than 0x10000
         space = self.space
         if space.isinstance_w(w_ob, space.w_unicode):
             u = space.unicode_w(w_ob)
-            if len(u) == 1:
-                u = ord(u[0])
-                if size == 2 and u > 0xffff:
+            try:
+                ordinal = wchar_helper.unicode_to_ordinal(u)
+            except ValueError:
+                pass
+            else:
+                if self.size == 2 and ordinal > 0xffff:
                     raise self._convert_error("single character <= 0xFFFF",
                                               w_ob)
-                return r_uint(u)
-            elif size == 4 and len(u) == 2 and ...
-
+                return ordinal
         elif (isinstance(w_ob, cdataobj.W_CData) and
                isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
-               w_ob.ctype.size == 2):
+               w_ob.ctype.size == self.size):
             with w_ob as ptr:
-                return misc.read_raw_ulong_data(ptr, 2)
+                return misc.read_raw_ulong_data(ptr, self.size)
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
-        ordinal = self._convert_to_char16(w_ob, self.size)
+        ordinal = self._convert_to_charN_t(w_ob)
         misc.write_raw_unsigned_data(cdata, ordinal, self.size)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
-        u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+        if self.size == 2:
+            u = wchar_helper.unicode_from_char16(ptr, length)
+        else:
+            try:
+                u = wchar_helper.unicode_from_char32(ptr, length)
+            except OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
         return self.space.newunicode(u)
 
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -91,11 +91,24 @@
             if not space.isinstance_w(w_ob, space.w_unicode):
                 raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
+            XXXXXXXXXXXXXXX
             n = len(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
+
+
+
+
+            if self.ctitem.size == 2:
+                length = wchar_helper.measure_length_16(ptr, length)
+            else:
+                length = wchar_helper.measure_length_32(ptr, length)
+            XXXX
+
+
+
             unichardata = rffi.cast(rffi.CWCHARP, cdata)
             copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
             if n != self.length:
@@ -134,12 +147,12 @@
                 #
                 # pointer to a wchar_t: builds and returns a unicode
                 if self.is_unichar_ptr_or_array():
-                    cdata = rffi.cast(rffi.CWCHARP, ptr)
-                    if length < 0:
-                        u = rffi.wcharp2unicode(cdata)
+                    from pypy.module._cffi_backend import wchar_helper
+                    if self.ctitem.size == 2:
+                        length = wchar_helper.measure_length_16(ptr, length)
                     else:
-                        u = rffi.wcharp2unicoden(cdata, length)
-                    return space.newunicode(u)
+                        length = wchar_helper.measure_length_32(ptr, length)
+                    return self.ctitem.unpack_ptr(self, ptr, length)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -304,6 +317,7 @@
             length = space.int_w(space.len(w_init))
         elif space.isinstance_w(w_init, space.w_basestring):
             # from a string, we add the null terminator
+            XXXXXXXXXXXXXXX
             length = space.int_w(space.len(w_init)) + 1
         elif self.is_file:
             result = self.prepare_file(w_init)
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -296,6 +296,7 @@
         return (w_value, space.int_w(space.len(w_value)))
     elif space.isinstance_w(w_value, space.w_basestring):
         # from a string, we add the null terminator
+        XXXXXXXXXX
         return (w_value, space.int_w(space.len(w_value)) + 1)
     else:
         explicitlength = space.getindex_w(w_value, space.w_OverflowError)
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -73,6 +73,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -505,6 +505,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -513,6 +514,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -80,8 +80,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1925,7 +1925,11 @@
         assert string(a, 8).startswith(b'ABC')  # may contain additional garbage
 
 def test_string_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     assert string(cast(BWChar, 42)) == u+'*'
     assert string(cast(BWChar, 0x4253)) == u+'\u4253'
     assert string(cast(BWChar, 0)) == u+'\x00'
@@ -2088,6 +2092,10 @@
 
 def test_wchar():
     _test_wchar_variant("wchar_t")
+    if sys.platform.startswith("linux"):
+        BWChar = new_primitive_type("wchar_t")
+        assert sizeof(BWChar) == 4
+        assert int(cast(BWChar, -1)) == -1        # signed, on linux
 
 def test_char16():
     BChar16 = new_primitive_type("char16_t")
@@ -2231,6 +2239,22 @@
         x = cast(BWChar, -1)
         py.test.raises(ValueError, string, x)
 
+def test_wchar_variants_mix():
+    BWChar  = new_primitive_type("wchar_t")
+    BChar16 = new_primitive_type("char16_t")
+    BChar32 = new_primitive_type("char32_t")
+    assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+    assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+    #
+    BChar16A = new_array_type(new_pointer_type(BChar16), None)
+    BChar32A = new_array_type(new_pointer_type(BChar32), None)
+    x = cast(BChar32, 'A')
+    py.test.raises(TypeError, newp, BChar16A, [x])
+    x = cast(BChar16, 'A')
+    py.test.raises(TypeError, newp, BChar32A, [x])
+
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
@@ -3457,14 +3481,15 @@
     py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
     py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
     #
-    BUniChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BUniChar), None)
-    p = newp(BArray, u+"foobar")
-    p[2:5] = [u+"*", u+"Z", u+"T"]
-    p[1:3] = u+"XY"
-    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
-    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
-    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BUniChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BUniChar), None)
+        p = newp(BArray, u+"foobar")
+        p[2:5] = [u+"*", u+"Z", u+"T"]
+        p[1:3] = u+"XY"
+        assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+        py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+        py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
 
 def test_void_p_arithmetic():
     BVoid = new_void_type()
@@ -3777,10 +3802,12 @@
     p0 = p
     assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
     assert unpack(p+1, 5) == b"bc\x00de"
-    BWChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
-    p = newp(BArray, u"abc\x00def")
-    assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BWChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
+        p = newp(BArray, u"abc\x00def")
+        assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
 
     for typename, samples in [
             ("uint8_t",  [0, 2**8-1]),
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
--- a/pypy/module/_cffi_backend/wchar_helper.py
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -1,7 +1,8 @@
+from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
 from rpython.rtyper.lltypesystem import lltype, rffi
 
-SIZE_UNICHAR = rffi.sizeof(lltype.UniChar)
+SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
 
 
 if SIZE_UNICODE == 4:
@@ -15,3 +16,102 @@
             ordinal = intmask(ordinal - 0x10000)
             return (unichr(0xD800 | (ordinal >> 10)) +
                     unichr(0xDC00 | (ordinal & 0x3FF)))
+
+def is_surrogate(u, index):
+    return (index + 1 < len(u) and
+            unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+            unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
+
+def as_surrogate(u, index):
+    ordinal = (ord(u[index + 0]) - 0xD800) << 10
+    ordinal |= (ord(u[index + 1]) - 0xDC00)
+    return r_uint(ordinal + 0x10000)
+
+def unicode_to_ordinal(u):
+    if len(u) == 1:
+        u = ord(u[0])
+        return r_uint(u)
+    elif SIZE_UNICODE == 2:
+        if len(u) == 2 and is_surrogate(u, 0):
+            return r_uint(as_surrogate(u, 0))
+    raise ValueError
+
+
+class OutOfRange(Exception):
+    def __init__(self, ordinal):
+        ordinal = intmask(rffi.cast(rffi.INT, ordinal))
+        self.ordinal = ordinal
+
+
+if SIZE_UNICODE == 2:
+    def unicode_from_char32(ptr, length):
+        ptr = rffi.cast(rffi.UINTP, ptr)
+        alloc = length
+        for i in range(length):
+            if rffi.cast(lltype.Unsigned, ptr[i]) > 0xFFFF:
+                alloc += 1
+
+        u = [u'\x00'] * alloc
+        j = 0
+        for i in range(length):
+            ordinal = rffi.cast(lltype.Unsigned, ptr[i])
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal = intmask(ordinal - 0x10000)
+                u[j] = unichr(0xD800 | (ordinal >> 10))
+                j += 1
+                u[j] = unichr(0xDC00 | (ordinal & 0x3FF))
+                j += 1
+            else:
+                u[j] = unichr(intmask(ordinal))
+                j += 1
+        assert j == len(u)
+        return u''.join(u)
+
+    def unicode_from_char16(ptr, length):
+        return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+else:
+    def unicode_from_char32(ptr, length):
+        return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+    def unicode_from_char16(ptr, length):
+        ptr = rffi.cast(rffi.USHORTP, ptr)
+        u = [u'\x00'] * length
+        i = 0
+        j = 0
+        while j < length:
+            ch = intmask(ptr[j])
+            j += 1
+            if 0xD800 <= ch <= 0xDBFF and j < length:
+                ch2 = intmask(ptr[j])
+                if 0xDC00 <= ch2 <= 0xDFFF:
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000
+                    j += 1
+            u[i] = unichr(ch)
+            i += 1
+        del u[i:]
+        return u''.join(u)
+
+
+ at specialize.ll()
+def _measure_length(ptr, maxlen):
+    result = 0
+    if maxlen < 0:
+        while intmask(ptr[result]) != 0:
+            result += 1
+    else:
+        while result < maxlen and intmask(ptr[result]) != 0:
+            result += 1
+    return result
+
+def measure_length_16(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.USHORTP, ptr), maxlen)
+
+def measure_length_32(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
+
+
+def unicode_to_char16(u, ptr):
+    XXX

From pypy.commits at gmail.com  Sun Jun  4 03:59:01 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 00:59:01 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: in-progress
Message-ID: <5933bdc5.508e1c0a.4b489.5669@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91504:8bc39f008ba8
Date: 2017-06-04 09:58 +0200
http://bitbucket.org/pypy/pypy/changeset/8bc39f008ba8/

Log:	in-progress

diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -36,8 +36,7 @@
         datasize = self.size
         #
         if datasize < 0:
-            from pypy.module._cffi_backend import misc
-            w_init, length = misc.get_new_array_length(space, w_init)
+            w_init, length = self.get_new_array_length(w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:
@@ -53,6 +52,29 @@
                 self.convert_from_object(ptr, w_init)
         return cdata
 
+    def get_new_array_length(self, w_value):
+        space = self.space
+        if (space.isinstance_w(w_value, space.w_list) or
+            space.isinstance_w(w_value, space.w_tuple)):
+            return (w_value, space.int_w(space.len(w_value)))
+        elif space.isinstance_w(w_value, space.w_bytes):
+            # from a string, we add the null terminator
+            s = space.bytes_w(w_value)
+            return (w_value, len(s) + 1)
+        elif space.isinstance_w(w_value, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_value)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            return (w_value, length + 1)
+        else:
+            explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+            if explicitlength < 0:
+                raise oefmt(space.w_ValueError, "negative array length")
+            return (space.w_None, explicitlength)
+
     def _check_subscript_index(self, w_cdata, i):
         space = self.space
         if i < 0:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -42,12 +42,13 @@
     def cast_unicode(self, w_ob):
         space = self.space
         s = space.unicode_w(w_ob)
-        XXXXXXXXXXXXXX
-        if len(s) != 1:
+        try:
+            ordinal = wchar_helper.unicode_to_ordinal(s)
+        except ValueError:
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return intmask(ordinal)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -4,9 +4,9 @@
 
 from rpython.rlib import rposix
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rtyper.annlowlevel import llstr, llunicode
+from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
@@ -88,31 +88,23 @@
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
+            from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
                 raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
-            XXXXXXXXXXXXXXX
-            n = len(s)
+            if self.ctitem.size == 2:
+                n = wchar_helper.unicode_size_as_char16(s)
+            else:
+                n = wchar_helper.unicode_size_as_char32(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-
-
-
-
+            add_final_zero = (n != self.length)
             if self.ctitem.size == 2:
-                length = wchar_helper.measure_length_16(ptr, length)
+                wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
             else:
-                length = wchar_helper.measure_length_32(ptr, length)
-            XXXX
-
-
-
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
-            if n != self.length:
-                unichardata[n] = u'\x00'
+                wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
@@ -315,10 +307,18 @@
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
-        elif space.isinstance_w(w_init, space.w_basestring):
+        elif space.isinstance_w(w_init, space.w_bytes):
             # from a string, we add the null terminator
-            XXXXXXXXXXXXXXX
-            length = space.int_w(space.len(w_init)) + 1
+            s = space.bytes_w(w_init)
+            length = len(s) + 1
+        elif space.isinstance_w(w_init, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_init)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            length += 1
         elif self.is_file:
             result = self.prepare_file(w_init)
             if result:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -244,7 +244,7 @@
         ct = self.ctype
         if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
             space = ct.space
-            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            w_ob, varsizelength = ct.get_new_array_length(w_ob)
             if optvarsize != -1:
                 # in this mode, the only purpose of this function is to compute
                 # the real size of the structure from a var-sized C99 array
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -290,22 +290,6 @@
 
 # ____________________________________________________________
 
-def get_new_array_length(space, w_value):
-    if (space.isinstance_w(w_value, space.w_list) or
-        space.isinstance_w(w_value, space.w_tuple)):
-        return (w_value, space.int_w(space.len(w_value)))
-    elif space.isinstance_w(w_value, space.w_basestring):
-        # from a string, we add the null terminator
-        XXXXXXXXXX
-        return (w_value, space.int_w(space.len(w_value)) + 1)
-    else:
-        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
-        if explicitlength < 0:
-            raise oefmt(space.w_ValueError, "negative array length")
-        return (space.w_None, explicitlength)
-
-# ____________________________________________________________
-
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
--- a/pypy/module/_cffi_backend/wchar_helper.py
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -1,6 +1,8 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.annlowlevel import llunicode
 from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_unicode_to_raw
 
 SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
 
@@ -18,8 +20,7 @@
                     unichr(0xDC00 | (ordinal & 0x3FF)))
 
 def is_surrogate(u, index):
-    return (index + 1 < len(u) and
-            unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+    return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
             unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
 
 def as_surrogate(u, index):
@@ -42,9 +43,13 @@
         ordinal = intmask(rffi.cast(rffi.INT, ordinal))
         self.ordinal = ordinal
 
+def _unicode_from_wchar(ptr, length):
+    return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
 
 if SIZE_UNICODE == 2:
     def unicode_from_char32(ptr, length):
+        # 'ptr' is a pointer to 'length' 32-bit integers
         ptr = rffi.cast(rffi.UINTP, ptr)
         alloc = length
         for i in range(length):
@@ -69,14 +74,13 @@
         assert j == len(u)
         return u''.join(u)
 
-    def unicode_from_char16(ptr, length):
-        return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+    unicode_from_char16 = _unicode_from_wchar
 
 else:
-    def unicode_from_char32(ptr, length):
-        return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+    unicode_from_char32 = _unicode_from_wchar
 
     def unicode_from_char16(ptr, length):
+        # 'ptr' is a pointer to 'length' 16-bit integers
         ptr = rffi.cast(rffi.USHORTP, ptr)
         u = [u'\x00'] * length
         i = 0
@@ -113,5 +117,71 @@
     return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
 
 
-def unicode_to_char16(u, ptr):
-    XXX
+def unicode_size_as_char16(u):
+    result = len(u)
+    if SIZE_UNICODE == 4:
+        for i in range(result):
+            if ord(u[i]) > 0xFFFF:
+                result += 1
+    return result
+
+def unicode_size_as_char32(u):
+    result = len(u)
+    if SIZE_UNICODE == 2 and result > 1:
+        for i in range(result - 1):
+            if is_surrogate(u, i):
+                result -= 1
+    return result
+
+
+def _unicode_to_wchar(u, target_ptr, target_length, add_final_zero):
+    # 'target_ptr' is a raw pointer to 'target_length' wchars;
+    # we assume here that target_length == len(u).
+    unichardata = rffi.cast(rffi.CWCHARP, target_ptr)
+    copy_unicode_to_raw(llunicode(u), unichardata, 0, target_length)
+    if add_final_zero:
+        unichardata[target_length] = u'\x00'
+
+
+if SIZE_UNICODE == 2:
+    def unicode_to_char32(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 32-bit integers;
+        # we assume here that target_length == unicode_size_as_char32(u).
+        ptr = rffi.cast(rffi.UINTP, target_ptr)
+        src_index = 0
+        for i in range(target_length):
+            if i < target_length - 1 and is_surrogate(u, src_index):
+                ordinal = as_surrogate(u, src_index)
+                src_index += 2
+            else:
+                ordinal = r_uint(ord(u[src_index]))
+                src_index += 1
+            ptr[i] = rffi.cast(rffi.UINT, ordinal)
+        if add_final_zero:
+            ptr[target_length] = rffi.cast(rffi.UINT, 0)
+
+    unicode_to_char16 = _unicode_to_wchar
+
+else:
+    unicode_to_char32 = _unicode_to_wchar
+
+    def unicode_to_char16(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 16-bit integers;
+        # we assume here that target_length == unicode_size_as_char16(u).
+        ptr = rffi.cast(rffi.USHORTP, target_ptr)
+        for uc in u:
+            ordinal = ord(uc)
+            if ordinal > 0xFFFF:
+                # NB. like CPython, ignore the problem of unicode string
+                # objects containing characters greater than sys.maxunicode
+                ordinal -= 0x10000
+                ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
+                ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))
+                ptr = rffi.ptradd(ptr, 2)
+            else:
+                ptr[0] = rffi.cast(rffi.USHORT, ordinal)
+                ptr = rffi.ptradd(ptr, 1)
+        assert ptr == (
+            rffi.ptradd(rffi.cast(rffi.USHORTP, target_ptr), target_length))
+        if add_final_zero:
+            ptr[0] = rffi.cast(rffi.USHORT, 0)

From pypy.commits at gmail.com  Sun Jun  4 04:07:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 01:07:42 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: Translation fixes
Message-ID: <5933bfce.15371c0a.42f9f.593e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91505:abfa8aff51d7
Date: 2017-06-04 10:06 +0200
http://bitbucket.org/pypy/pypy/changeset/abfa8aff51d7/

Log:	Translation fixes

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -218,7 +218,7 @@
         else:
             try:
                 u = wchar_helper.unicode_from_char32(ptr, length)
-            except OutOfRange as e:
+            except wchar_helper.OutOfRange as e:
                 raise oefmt(self.space.w_ValueError,
                             "char32_t out of range for "
                             "conversion to unicode: %s", hex(e.ordinal))
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
--- a/pypy/module/_cffi_backend/wchar_helper.py
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -39,6 +39,8 @@
 
 
 class OutOfRange(Exception):
+    ordinal = 0
+
     def __init__(self, ordinal):
         ordinal = intmask(rffi.cast(rffi.INT, ordinal))
         self.ordinal = ordinal

From pypy.commits at gmail.com  Sun Jun  4 04:27:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 01:27:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix test on 32-bit
Message-ID: <5933c47e.0d931c0a.5051e.8706@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91506:e2ba7d32b791
Date: 2017-06-04 10:27 +0200
http://bitbucket.org/pypy/pypy/changeset/e2ba7d32b791/

Log:	Fix test on 32-bit

diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -33,7 +33,7 @@
         add_loop_instructions = ('mov; '
                                  'lea; '    # a nop, for the label
                                  'add; test; je; jmp;')   # plus some padding
-        bridge_loop_instructions = 'cmp; jge; mov; mov; call; jmp;'
+        bridge_loop_instructions = 'cmp; jl; jmp;'
     else:
         add_loop_instructions = ('mov; '
                                  'nop; '    # for the label

From pypy.commits at gmail.com  Sun Jun  4 04:30:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 01:30:25 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: import cffi/6c465c147687
Message-ID: <5933c521.d1d71c0a.1a6d1.e428@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91507:b061a2e969ec
Date: 2017-06-04 10:29 +0200
http://bitbucket.org/pypy/pypy/changeset/b061a2e969ec/

Log:	import cffi/6c465c147687

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -62,11 +62,16 @@
     typedef unsigned char _Bool;
 #  endif
 # endif
+# if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
+    typedef uint16_t char16_t;
+    typedef uint32_t char32_t;
+# endif
 #else
 # include <stdint.h>
 # if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
+# include <uchar.h>
 #endif
 
 #ifdef __GNUC__
@@ -159,9 +164,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +179,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +224,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -2254,6 +2254,11 @@
     py.test.raises(TypeError, newp, BChar16A, [x])
     x = cast(BChar16, 'A')
     py.test.raises(TypeError, newp, BChar32A, [x])
+    #
+    a = newp(BChar16A, u+'\U00012345')
+    assert len(a) == 3
+    a = newp(BChar32A, u+'\U00012345')
+    assert len(a) == 2   # even if the Python unicode string above is 2 chars
 
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -2,6 +2,7 @@
 import py, sys, platform
 import pytest
 from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 from cffi import FFI
 import _cffi_backend
 
@@ -398,6 +399,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -509,3 +512,43 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+    def test_explicitly_defined_char16_t(self):
+        ffi = FFI()
+        ffi.cdef("typedef uint16_t char16_t;")
+        x = ffi.cast("char16_t", 1234)
+        assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+    def test_char16_t(self):
+        ffi = FFI()
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        ffi = FFI()
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        py_uni = u+'\U00012345'
+        z = ffi.new("char32_t[]", py_uni)
+        assert len(z) == 2
+        assert list(z) == [py_uni, u+'\x00']    # maybe a 2-unichars string
+        assert ffi.string(z) == py_uni
+        if len(py_uni) == 1:    # 4-bytes unicodes in Python
+            s = ffi.new("char32_t[]", u+'\ud808\udf00')
+            assert len(s) == 3
+            assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -3,6 +3,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 
 
 SOURCE = """\
@@ -93,6 +94,15 @@
 }
 
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+    return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+    return (unsigned int)(a + 42);
+}
 """
 
 class TestOwnLib(object):
@@ -301,3 +311,18 @@
         pfn = ffi.addressof(lib, "test_getting_errno")
         assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
         assert pfn == lib.test_getting_errno
+
+    def test_char16_char32_t(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            char16_t foo_2bytes(char16_t);
+            char32_t foo_4bytes(char32_t);
+        """)
+        lib = ffi.dlopen(self.module)
+        assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -242,7 +242,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true
@@ -385,6 +385,10 @@
     lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
     assert lib.foo(uniexample1) == uniexample2
 
+def test_char16_char32_type():
+    py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+                 "working in ffi.verify() mode")
+
 def test_no_argument():
     ffi = FFI()
     ffi.cdef("int foo(void);")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1673,6 +1673,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -1743,3 +1745,30 @@
         exec("from _test_import_from_lib import *", d)
         assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
                 ['ffi', 'lib'])
+
+    def test_char16_t(self):
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,13 +25,14 @@
     assert ''.join(map(str, recomp.cffi_types)) == expected_output
 
 def verify(ffi, module_name, source, *args, **kwds):
+    no_cpp = kwds.pop('no_cpp', False)
     kwds.setdefault('undef_macros', ['NDEBUG'])
     module_name = '_CFFI_' + module_name
     ffi.set_source(module_name, source)
-    if not os.environ.get('NO_CPP'):     # test the .cpp mode too
+    if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
-    else:
+    elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
@@ -2251,3 +2252,29 @@
     int f(int a) { return a + 40; }
     """, extra_compile_args=['-fvisibility=hidden'])
     assert lib.f(2) == 42
+
+def test_override_default_definition():
+    ffi = FFI()
+    ffi.cdef("typedef long int16_t, char16_t;")
+    lib = verify(ffi, "test_override_default_definition", "")
+    assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+    ffi = FFI()
+    ffi.cdef("""
+        char16_t foo_2bytes(char16_t);
+        char32_t foo_4bytes(char32_t);
+    """)
+    lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+    char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+    """, no_cpp=no_cpp)
+    assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+    py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+    py.test.raises(TypeError, lib.foo_2bytes, 1234)
+    py.test.raises(TypeError, lib.foo_4bytes, 1234)
+
+def test_char16_char32_plain_c():
+    test_char16_char32_type(no_cpp=True)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -222,7 +222,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true

From pypy.commits at gmail.com  Sun Jun  4 06:33:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 03:33:56 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: Bump the internal version max
Message-ID: <5933e214.81851c0a.31077.68e3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91508:113befadf44a
Date: 2017-06-04 12:33 +0200
http://bitbucket.org/pypy/pypy/changeset/113befadf44a/

Log:	Bump the internal version max

diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,7 +9,7 @@
 
 
 VERSION_MIN    = 0x2601
-VERSION_MAX    = 0x27FF
+VERSION_MAX    = 0x28FF
 
 VERSION_EXPORT = 0x0A03
 

From pypy.commits at gmail.com  Sun Jun  4 07:28:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 04:28:46 -0700 (PDT)
Subject: [pypy-commit] cffi default: extra test
Message-ID: <5933eeee.44bf1c0a.62dd4.1138@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2971:a5114b7d3887
Date: 2017-06-04 13:28 +0200
http://bitbucket.org/cffi/cffi/changeset/a5114b7d3887/

Log:	extra test

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -2265,6 +2265,11 @@
     py.test.raises(TypeError, newp, BChar16A, [x])
     x = cast(BChar16, 'A')
     py.test.raises(TypeError, newp, BChar32A, [x])
+    #
+    a = newp(BChar16A, u+'\U00012345')
+    assert len(a) == 3
+    a = newp(BChar32A, u+'\U00012345')
+    assert len(a) == 2   # even if the Python unicode string above is 2 chars
 
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a

From pypy.commits at gmail.com  Sun Jun  4 08:24:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 05:24:43 -0700 (PDT)
Subject: [pypy-commit] cffi default: Detect and complain when trying to
 convert a char32_t to unicode if
Message-ID: <5933fc0b.23afdf0a.136b8.0ce7@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2972:0300a91f7fb4
Date: 2017-06-04 14:24 +0200
http://bitbucket.org/cffi/cffi/changeset/0300a91f7fb4/

Log:	Detect and complain when trying to convert a char32_t to unicode if
	the unicode uses 16-bit chars and the original char32_t is out of
	range even for surrogates

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1390,10 +1390,9 @@
             if (n != ct->ct_length)
                 n++;
             if (ctitem->ct_size == 4)
-                _my_PyUnicode_AsChar32(init, (cffi_char32_t *)data, n);
+                return _my_PyUnicode_AsChar32(init, (cffi_char32_t *)data, n);
             else
-                _my_PyUnicode_AsChar16(init, (cffi_char16_t *)data, n);
-            return 0;
+                return _my_PyUnicode_AsChar16(init, (cffi_char16_t *)data, n);
         }
     }
     else {
diff --git a/c/wchar_helper.h b/c/wchar_helper.h
--- a/c/wchar_helper.h
+++ b/c/wchar_helper.h
@@ -195,9 +195,9 @@
     return result;
 }
 
-static void _my_PyUnicode_AsChar16(PyObject *unicode,
-                                   cffi_char16_t *result,
-                                   Py_ssize_t resultlen)
+static int _my_PyUnicode_AsChar16(PyObject *unicode,
+                                  cffi_char16_t *result,
+                                  Py_ssize_t resultlen)
 {
     Py_ssize_t len = PyUnicode_GET_SIZE(unicode);
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
@@ -208,9 +208,12 @@
 #else
         cffi_char32_t ordinal = u[i];
         if (ordinal > 0xFFFF) {
-            /* NB. like CPython, ignore the problem of unicode string objects
-             * containing characters greater than sys.maxunicode.  It is
-             * easier to not add exception handling here */
+            if (ordinal > 0x10FFFF) {
+                PyErr_Format(PyExc_ValueError,
+                             "unicode character out of range for "
+                             "conversion to char16_t: 0x%x", (int)ordinal);
+                return -1;
+            }
             ordinal -= 0x10000;
             *result++ = 0xD800 | (ordinal >> 10);
             *result++ = 0xDC00 | (ordinal & 0x3FF);
@@ -219,11 +222,12 @@
 #endif
         *result++ = ordinal;
     }
+    return 0;
 }
 
-static void _my_PyUnicode_AsChar32(PyObject *unicode,
-                                   cffi_char32_t *result,
-                                   Py_ssize_t resultlen)
+static int _my_PyUnicode_AsChar32(PyObject *unicode,
+                                  cffi_char32_t *result,
+                                  Py_ssize_t resultlen)
 {
     Py_UNICODE *u = PyUnicode_AS_UNICODE(unicode);
     Py_ssize_t i;
@@ -238,4 +242,5 @@
         result[i] = ordinal;
         u++;
     }
+    return 0;
 }

From pypy.commits at gmail.com  Sun Jun  4 08:26:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 05:26:18 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: Detect and complain about
 unicode "characters" that are greater than
Message-ID: <5933fc6a.5aa0df0a.1cbcc.05b0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91509:649a8f742c90
Date: 2017-06-04 14:25 +0200
http://bitbucket.org/pypy/pypy/changeset/649a8f742c90/

Log:	Detect and complain about unicode "characters" that are greater than
	0x10FFFF when attempting to convert to a pair of surrogates

diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -179,7 +179,12 @@
             return self.space.newunicode(unichardata[0])
         else:
             value = misc.read_raw_ulong_data(cdata, self.size)   # r_uint
-            u = wchar_helper.ordinal_to_unicode(value)
+            try:
+                u = wchar_helper.ordinal_to_unicode(value)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
             return self.space.newunicode(u)
 
     def string(self, cdataobj, maxlen):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -102,7 +102,12 @@
                             "(got %d characters)", self.name, n)
             add_final_zero = (n != self.length)
             if self.ctitem.size == 2:
-                wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+                try:
+                    wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+                except wchar_helper.OutOfRange as e:
+                    raise oefmt(self.space.w_ValueError,
+                                "unicode character ouf of range for "
+                                "conversion to char16_t: %s", hex(e.ordinal))
             else:
                 wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
--- a/pypy/module/_cffi_backend/wchar_helper.py
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -14,10 +14,12 @@
     def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
         if ordinal <= 0xffff:
             return unichr(intmask(ordinal))
-        else:
+        elif ordinal <= 0x10ffff:
             ordinal = intmask(ordinal - 0x10000)
             return (unichr(0xD800 | (ordinal >> 10)) +
                     unichr(0xDC00 | (ordinal & 0x3FF)))
+        else:
+            raise OutOfRange(ordinal)
 
 def is_surrogate(u, index):
     return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
@@ -174,8 +176,8 @@
         for uc in u:
             ordinal = ord(uc)
             if ordinal > 0xFFFF:
-                # NB. like CPython, ignore the problem of unicode string
-                # objects containing characters greater than sys.maxunicode
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
                 ordinal -= 0x10000
                 ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
                 ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))

From pypy.commits at gmail.com  Sun Jun  4 08:40:26 2017
From: pypy.commits at gmail.com (mjacob)
Date: Sun, 04 Jun 2017 05:40:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5933ffba.e9afdf0a.dee31.064d@mx.google.com>

Author: Manuel Jacob <me at manueljacob.de>
Branch: py3.5
Changeset: r91510:2bc0ce15d8d0
Date: 2017-06-04 14:39 +0200
http://bitbucket.org/pypy/pypy/changeset/2bc0ce15d8d0/

Log:	hg merge default

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -8,25 +8,23 @@
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
-PyPy3.5 (our first in the 3.5 series) includes the upstream stdlib version
-3.5.3.
+PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-We continue to make incremental improvements to our C-API
-compatibility layer (cpyext). PyPy2 can now import and run many C-extension
-packages, among the most notable are Numpy, Cython, and Pandas. Performance may
-be slower than CPython, especially for frequently-called short C functions.
+This release enables `profile guided optimization` of the base interpreter,
+which may make unjitted code run faster.
+
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
 Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our first 3.5.3 beta
+version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
 release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
 as `these benchmarks show`_ it already gives a nice speed bump.
 We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
 "PyPy3.5" supports the Python 3.5 language).
 
-CFFI_ has been updated to 1.10, improving an already great package for
-interfacing with C.
+CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.10.1,
+improving an already great package for interfacing with C.
 
 As always, this release fixed many issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
@@ -44,6 +42,7 @@
 improvements, tweaking popular `modules`_ to run on pypy, or general `help`_
 with making RPython's JIT even better.
 
+.. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html
@@ -81,28 +80,52 @@
 
 See also issues that were resolved_
 
+Note that these are also merged into PyPy 3.5
+
 * New features and cleanups
 
-  * Implement PyModule_New, 
+  * Implement PyModule_New, Py_GetRecursionLimit, Py_SetRecursionLimit,
+    Py_EnterRecursiveCall, Py_LeaveRecursiveCall, populate tp_descr_get and
+    tp_descr_set slots,
+    add conversions of ``__len__``, ``__setitem__``, ``__delitem__`` to
+    appropriate C-API slots
   * Fix for multiple inheritance in app-level for C-API defined classes
   * Revert a change that removed tp_getattr (Part of the 5.7.1 bugfix release)
   * Document more differences with CPython here_
   * Add native PyPy support to profile frames in vmprof
   * Fix an issue with Exception order on failed import
   * Fix for a corner case of __future__ imports
+  * Update packaged Windows zlib, sqlite, expat and OpenSSL to versions used
+    by CPython
+  * Allow windows builds to use ``jom.exe`` for compiling in parallel
+  * Rewrite ``itertools.groupby()``, following CPython
+  * Backport changes from PyPy 3.5 to minimize the code differences
+  * Improve support for BSD using patches contributed by downstream
+  * Support profile-guided optimization, enabled with --profopt, , and
+    specify training data ``profoptpath``
 
-* Bug Fixes
+* Bug Fixes 
 
   * Correctly handle dict.pop where the popping key is not the same type as the
     dict's and pop is called with a default (Part of the 5.7.1 bugfix release)
   * Improve our file's universal newline .readline implementation for
     ``\n``, ``\r`` confusion
+  * Tweak issue where ctype array ``_base`` was set on empty arrays, now it
+    is closer to the implementation in CPython
+  * Fix critical bugs in shadowstack that crashed multithreaded programs and
+    very rarely showed up even in single threaded programs
+  * Remove flaky fastpath function call from ctypes
+  * Support passing a buffersize of 0 to socket.getsockopt
+  * Avoid hash() returning -1 in cpyext
 
 * Performance improvements:
 
   * Tweaks made to improve performance by reducing the number of guards
     inserted in jitted code, based on feedback from users
   * Add garbage collector memory pressure to some c-level allocations
+  * Speed up struck.pack, struck.pack_into
+  * Performance tweaks to round(x, n) for the case n == 0
+  * Improve zipfile performance by not doing repeated string concatenation
 
 * RPython improvements
 
@@ -119,6 +142,11 @@
     blocks are moved off-line.  Also, the temporary register used to contain
     large constants is reused across instructions. This helps CPUs branch
     predictor
+  * Refactor rpython.rtyper.controllerentry to use use ``@specialize`` instead
+    of ``._annspecialcase_``
+  * Refactor handling of buffers and memoryviews. Memoryviews will now be
+    accepted in a few more places, e.g. in compile()
+
 
 .. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
 
@@ -129,6 +157,15 @@
 
   * Implement main part of PEP 489 (multi-phase extension module initialization)
   * Add docstrings to various modules and functions
+  * Adapt many CPython bug/feature fixes from CPython 3.5 to PyPy3.5
+  * Translation succeeds on Mac OS X, unfortunately our buildbot slave cannot
+    be updated to the proper development versions of OpenSSL to properly
+    package a release.
+  * Implement `` _SSLSocket.server_side``
+  * Do not silently ignore ``_swappedbytes_`` in ctypes. We now raise a
+    ``NotImplementedError``
+  * Implement and expose ``msvcrt.SetErrorMode``
+  * Implement ``PyModule_GetState``
 
 * Bug Fixes
 
@@ -137,12 +174,21 @@
   * OSError(None,None) is different from OSError()
   * Get closer to supporting 32 bit windows, translation now succeeds and most
     lib-python/3/test runs
+  * Call ``sys.__interactivehook__`` at startup
+  * Let ``OrderedDict.__init__`` behave like CPython wrt. subclasses
+    overridding ``__setitem__``
 
 * Performance improvements:
 
   * Use "<python> -m test" to run the CPython test suite, as documented by CPython,
     instead of our outdated regrverbose.py script
   * Change _cffi_src/openssl/callbacks.py to stop relying on the CPython C API.
+  * Avoid importing the full locale module during _io initialization, 
+    CPython change fbbf8b160e8d
+  * Avoid freezing many app-level modules at translation, avoid importing many
+    modules at startup
+  * Refactor buffers, which allows an optimization for 
+    ``bytearray()[:n].tobytes()``
 
 * The following features of Python 3.5 are not implemented yet in PyPy:
 
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -18,31 +18,25 @@
             else:
                 self.builder = builder_cls(size)
 
-        def _check_done(self, space):
-            if self.builder is None:
-                raise oefmt(space.w_ValueError,
-                            "Can't operate on a built builder")
-
         @unwrap_spec(size=int)
         def descr__new__(space, w_subtype, size=-1):
             return W_Builder(space, size)
 
         @unwrap_spec(s=unwrap)
         def descr_append(self, space, s):
-            self._check_done(space)
             self.builder.append(s)
 
         @unwrap_spec(s=unwrap, start=int, end=int)
         def descr_append_slice(self, space, s, start, end):
-            self._check_done(space)
             if not 0 <= start <= end <= len(s):
                 raise oefmt(space.w_ValueError, "bad start/stop")
             self.builder.append_slice(s, start, end)
 
         def descr_build(self, space):
-            self._check_done(space)
             s = self.builder.build()
-            self.builder = None
+            # after build(), we can continue to append more strings
+            # to the same builder.  This is supported since
+            # 2ff5087aca28 in RPython.
             if strtype is str:
                 return space.newbytes(s)
             else:
diff --git a/pypy/module/__pypy__/test/test_builders.py b/pypy/module/__pypy__/test/test_builders.py
--- a/pypy/module/__pypy__/test/test_builders.py
+++ b/pypy/module/__pypy__/test/test_builders.py
@@ -9,8 +9,9 @@
         b.append("1")
         s = b.build()
         assert s == "abc1231"
-        raises(ValueError, b.build)
-        raises(ValueError, b.append, "123")
+        assert b.build() == s
+        b.append("123")
+        assert b.build() == s + "123"
 
     def test_preallocate(self):
         from __pypy__.builders import StringBuilder
@@ -27,7 +28,8 @@
         raises(ValueError, b.append_slice, "1", 2, 1)
         s = b.build()
         assert s == "cde"
-        raises(ValueError, b.append_slice, "abc", 1, 2)
+        b.append_slice("abc", 1, 2)
+        assert b.build() == "cdeb"
 
     def test_stringbuilder(self):
         from __pypy__.builders import BytesBuilder
@@ -37,6 +39,6 @@
         assert len(b) == 6
         b.append(b"you and me")
         s = b.build()
-        raises(ValueError, len, b)
+        assert len(b) == 16
         assert s == b"abc123you and me"
-        raises(ValueError, b.build)
+        assert b.build() == s
diff --git a/rpython/jit/backend/x86/test/test_runner.py b/rpython/jit/backend/x86/test/test_runner.py
--- a/rpython/jit/backend/x86/test/test_runner.py
+++ b/rpython/jit/backend/x86/test/test_runner.py
@@ -33,7 +33,7 @@
         add_loop_instructions = ('mov; '
                                  'lea; '    # a nop, for the label
                                  'add; test; je; jmp;')   # plus some padding
-        bridge_loop_instructions = 'cmp; jge; mov; mov; call; jmp;'
+        bridge_loop_instructions = 'cmp; jl; jmp;'
     else:
         add_loop_instructions = ('mov; '
                                  'nop; '    # for the label

From pypy.commits at gmail.com  Sun Jun  4 09:06:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 06:06:17 -0700 (PDT)
Subject: [pypy-commit] cffi default: Figured out a memory leak in an error
 case
Message-ID: <593405c9.5497df0a.d11b5.638c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2973:6e1b6c8e1d79
Date: 2017-06-04 15:06 +0200
http://bitbucket.org/cffi/cffi/changeset/6e1b6c8e1d79/

Log:	Figured out a memory leak in an error case

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -4137,7 +4137,10 @@
 
     assert(x->ct_unique_key == NULL);
     x->ct_unique_key = key; /* the key will be freed in ctypedescr_dealloc() */
-    Py_DECREF(x);          /* the 'value' in unique_cache doesn't count as 1 */
+    /* the 'value' in unique_cache doesn't count as 1, but don't use
+       Py_DECREF(x) here because it will confuse debug builds into thinking
+       there was an extra DECREF in total. */
+    ((PyObject *)x)->ob_refcnt--;
     return (PyObject *)x;
 
  error:
@@ -6231,6 +6234,7 @@
     src = cd->c_data;
     itemsize = ctitem->ct_size;
     if (itemsize < 0) {
+        Py_DECREF(result);
         PyErr_Format(PyExc_ValueError, "'%s' points to items of unknown size",
                      cd->c_type->ct_name);
         return NULL;

From pypy.commits at gmail.com  Sun Jun  4 09:37:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 06:37:14 -0700 (PDT)
Subject: [pypy-commit] cffi default: Another missing Py_DECREF
Message-ID: <59340d0a.2899df0a.c366.838c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2974:19e44e15058a
Date: 2017-06-04 15:37 +0200
http://bitbucket.org/cffi/cffi/changeset/19e44e15058a/

Log:	Another missing Py_DECREF

diff --git a/c/realize_c_type.c b/c/realize_c_type.c
--- a/c/realize_c_type.c
+++ b/c/realize_c_type.c
@@ -269,8 +269,11 @@
     PyObject *x = realize_c_type_or_func(builder, opcodes, index);
     if (x == NULL || CTypeDescr_Check(x))
         return (CTypeDescrObject *)x;
-    else
-        return unexpected_fn_type(x);
+    else {
+        unexpected_fn_type(x);
+        Py_DECREF(x);
+        return NULL;
+    }
 }
 
 static void _realize_name(char *target, const char *prefix, const char *srcname)

From pypy.commits at gmail.com  Sun Jun  4 10:19:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 07:19:42 -0700 (PDT)
Subject: [pypy-commit] cffi default: Didn't figure out how to cleanly define
 charN_t inside _cffi_include.h.
Message-ID: <593416fe.9199df0a.fedc3.cc08@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2975:5f90dcd1ce55
Date: 2017-06-04 16:18 +0200
http://bitbucket.org/cffi/cffi/changeset/5f90dcd1ce55/

Log:	Didn't figure out how to cleanly define charN_t inside
	_cffi_include.h. So for now, don't.

diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -62,16 +62,11 @@
     typedef unsigned char _Bool;
 #  endif
 # endif
-# if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
-    typedef uint16_t char16_t;
-    typedef uint32_t char32_t;
-# endif
 #else
 # include <stdint.h>
 # if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
-# include <uchar.h>
 #endif
 
 #ifdef __GNUC__
diff --git a/cffi/vengine_cpy.py b/cffi/vengine_cpy.py
--- a/cffi/vengine_cpy.py
+++ b/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/cffi/vengine_gen.py b/cffi/vengine_gen.py
--- a/cffi/vengine_gen.py
+++ b/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/doc/source/ref.rst b/doc/source/ref.rst
--- a/doc/source/ref.rst
+++ b/doc/source/ref.rst
@@ -629,9 +629,9 @@
 |               | or another <cdata char>| length 1         | ``<``          |
 +---------------+------------------------+------------------+----------------+
 | ``wchar_t``,  | a unicode of length 1  | a unicode of     |                |
-| ``char16_t``, | (or maybe 2 if         | length 1         | int() `[8]`,   |
+| ``char16_t``, | (or maybe 2 if         | length 1         | int(),         |
 | ``char32_t``  | surrogates) or         | (or maybe 2 if   | bool(), ``<``  |
-|               | another similar <cdata>| surrogates)      |                |
+| `[8]`         | another similar <cdata>| surrogates)      |                |
 +---------------+------------------------+------------------+----------------+
 |  ``float``,   | a float or anything on | a Python float   | float(), int(),|
 |  ``double``   | which float() works    |                  | bool(), ``<``  |
@@ -774,12 +774,22 @@
    take directly as argument types or return type a complex type cannot
    be called by CFFI, unless they are directly using the API mode.
 
-`[8]` sign of ``wchar_t``, ``char16_t`` and ``char32_t``
+`[8]` ``wchar_t``, ``char16_t`` and ``char32_t``
 
    The ``wchar_t`` type has the same signedness as the underlying
    platform's.  For example, on Linux, it is a signed 32-bit integer.
    However, the types ``char16_t`` and ``char32_t`` (*new in version
-   1.11*) are always unsigned.
+   1.11*) are always unsigned.  **Warning:** for now, if you use
+   ``char16_t`` and ``char32_t`` with ``cdef()`` and ``set_source()``,
+   you have to make sure yourself that the types are declared by the C
+   source you provide to ``set_source()``.  They would be declared if
+   you ``#include`` a library that explicitly uses them, for example,
+   or when using C++11.  Otherwise, you need ``#include <uchar.h>`` on
+   Linux, or more generally something like ``typedef uint_least16_t
+   char16_t;``.  This is not done automatically by CFFI because
+   ``uchar.h`` is not standard across platforms, and writing a
+   ``typedef`` like above would crash if the type happens to be
+   already defined.
 
 .. _file:
 
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -11,7 +11,9 @@
   when used as ``charN_t *`` or ``charN_t[]`` they represent a unicode
   string.  The difference with ``wchar_t`` is that they have a known,
   fixed size.  They should work at all places that used to work with
-  ``wchar_t`` (please report an issue if I missing something).
+  ``wchar_t`` (please report an issue if I missing something).  Note
+  that with ``set_source()``, you need to make sure that these types are
+  actually defined by the C source you provide (if used in ``cdef()``).
 
 * Support the C99 types ``float _Complex`` and ``double _Complex``.
   Note that libffi doesn't support them, which means that in the ABI
diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2265,6 +2265,11 @@
         char32_t foo_4bytes(char32_t);
     """)
     lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    #if !defined(__cplusplus) || __cplusplus < 201103L
+    typedef uint_least16_t char16_t;
+    typedef uint_least32_t char32_t;
+    #endif
+    
     char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
     char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
     """, no_cpp=no_cpp)

From pypy.commits at gmail.com  Sun Jun  4 10:20:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 07:20:50 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: import cffi/5f90dcd1ce55
Message-ID: <59341742.918bdf0a.d7179.006a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91511:c7b24a4c9b2d
Date: 2017-06-04 16:20 +0200
http://bitbucket.org/pypy/pypy/changeset/c7b24a4c9b2d/

Log:	import cffi/5f90dcd1ce55

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -62,16 +62,11 @@
     typedef unsigned char _Bool;
 #  endif
 # endif
-# if _MSC_VER < 1900 || !defined(__cplusplus)   /* MSVC < 2015, or plain C */
-    typedef uint16_t char16_t;
-    typedef uint32_t char32_t;
-# endif
 #else
 # include <stdint.h>
 # if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
-# include <uchar.h>
 #endif
 
 #ifdef __GNUC__
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2266,6 +2266,11 @@
         char32_t foo_4bytes(char32_t);
     """)
     lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    #if !defined(__cplusplus) || __cplusplus < 201103L
+    typedef uint_least16_t char16_t;
+    typedef uint_least32_t char32_t;
+    #endif
+    
     char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
     char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
     """, no_cpp=no_cpp)

From pypy.commits at gmail.com  Sun Jun  4 15:49:00 2017
From: pypy.commits at gmail.com (alex_gaynor)
Date: Sun, 04 Jun 2017 12:49:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Backport
 5de3a64179bafcd440b32849b1129ed1fea47b85 from CPython.
Message-ID: <5934642c.88a6df0a.66150.d61a@mx.google.com>

Author: Alex Gaynor <alex.gaynor at gmail.com>
Branch: 
Changeset: r91512:71f3e458447f
Date: 2017-06-04 15:45 -0400
http://bitbucket.org/pypy/pypy/changeset/71f3e458447f/

Log:	Backport 5de3a64179bafcd440b32849b1129ed1fea47b85 from CPython.

	This will speed up usage of the warning module considerably

diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
 
     def __init__(self, message, category, filename, lineno, file=None,
                     line=None):
-        local_values = locals()
-        for attr in self._WARNING_DETAILS:
-            setattr(self, attr, local_values[attr])
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
         self._category_name = category.__name__ if category else None
 
     def __str__(self):

From pypy.commits at gmail.com  Sun Jun  4 16:05:57 2017
From: pypy.commits at gmail.com (stefanor)
Date: Sun, 04 Jun 2017 13:05:57 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: pypy3 manpage
Message-ID: <59346825.d78adf0a.cca2.2c52@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-pypy3.5-5.x
Changeset: r91513:ed6022df8666
Date: 2017-06-04 23:05 +0300
http://bitbucket.org/pypy/pypy/changeset/ed6022df8666/

Log:	pypy3 manpage (grafted from
	e00caafaf849ac80d2aeb16b3fc6803f6526719c)

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -222,8 +222,8 @@
 # -- Options for manpage output-------------------------------------------------
 
 man_pages = [
-  ('man/pypy.1', 'pypy',
-   u'fast, compliant alternative implementation of the Python language',
+  ('man/pypy3.1', 'pypy3',
+   u'fast, compliant alternative implementation of the Python 3 language',
    u'The PyPy Project', 1)
 ]
 
diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy3.1.rst
copy from pypy/doc/man/pypy.1.rst
copy to pypy/doc/man/pypy3.1.rst
--- a/pypy/doc/man/pypy.1.rst
+++ b/pypy/doc/man/pypy3.1.rst
@@ -1,14 +1,14 @@
-======
- pypy
-======
+=======
+ pypy3
+=======
 
-.. note: this is turned into a regular man page "pypy.1" by
+.. note: this is turned into a regular man page "pypy3.1" by
    doing "make man" in pypy/doc/
 
 SYNOPSIS
 ========
 
-``pypy`` [*options*]
+``pypy3`` [*options*]
 [``-c`` *cmd*\ \|\ ``-m`` *mod*\ \|\ *file.py*\ \|\ ``-``\ ]
 [*arg*\ ...]
 
@@ -69,7 +69,7 @@
 ===========
 
 ``PYTHONPATH``
-    Add directories to pypy's module search path.
+    Add directories to pypy3's module search path.
     The format is the same as shell's ``PATH``.
 
 ``PYTHONSTARTUP``
@@ -132,4 +132,4 @@
 SEE ALSO
 ========
 
-**python**\ (1)
+**python3**\ (1)

From pypy.commits at gmail.com  Sun Jun  4 16:05:59 2017
From: pypy.commits at gmail.com (stefanor)
Date: Sun, 04 Jun 2017 13:05:59 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: Expose SOABI in sysconfig
 in pypy3 too
Message-ID: <59346827.44bf1c0a.62dd4.c0bc@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: release-pypy3.5-5.x
Changeset: r91514:81df83380eea
Date: 2017-06-04 23:05 +0300
http://bitbucket.org/pypy/pypy/changeset/81df83380eea/

Log:	Expose SOABI in sysconfig in pypy3 too (grafted from
	228f8b4206aea43feb6621ed1df7db94d0507ebf)

diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
--- a/lib_pypy/_sysconfigdata.py
+++ b/lib_pypy/_sysconfigdata.py
@@ -5,5 +5,6 @@
 build_time_vars = {
     "EXT_SUFFIX": so_ext,
     "SHLIB_SUFFIX": so_ext,
+    "SOABI": '-'.join(so_ext.split('.')[1].split('-')[:2]),
     "SO": so_ext  # deprecated in Python 3, for backward compatibility
 }

From pypy.commits at gmail.com  Sun Jun  4 17:39:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 14:39:03 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: Copy failing test from the
 CPython cffi test suite; fix it (for 16-bit
Message-ID: <59347df7.c9d91c0a.c5014.2591@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91515:35d3643b476a
Date: 2017-06-04 23:38 +0200
http://bitbucket.org/pypy/pypy/changeset/35d3643b476a/

Log:	Copy failing test from the CPython cffi test suite; fix it (for
	16-bit unicode chars)

diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -555,3 +555,11 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         raises(ffi.error, ffi.cast, "int[-5]", 0)
+
+    def test_char32_t(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        z = ffi.new("char32_t[]", u'\U00012345')
+        assert len(z) == 2
+        assert ffi.cast("int *", z)[0] == 0x12345
+        assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
--- a/pypy/module/_cffi_backend/wchar_helper.py
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -153,8 +153,9 @@
         # we assume here that target_length == unicode_size_as_char32(u).
         ptr = rffi.cast(rffi.UINTP, target_ptr)
         src_index = 0
+        last_surrogate_pos = len(u) - 2
         for i in range(target_length):
-            if i < target_length - 1 and is_surrogate(u, src_index):
+            if src_index <= last_surrogate_pos and is_surrogate(u, src_index):
                 ordinal = as_surrogate(u, src_index)
                 src_index += 2
             else:

From pypy.commits at gmail.com  Sun Jun  4 17:43:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 14:43:17 -0700 (PDT)
Subject: [pypy-commit] cffi default: including <complex.h> fails on some
 systems with C++
Message-ID: <59347ef5.4495df0a.12a18.9f22@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2976:e7063ce4bdf8
Date: 2017-06-04 23:43 +0200
http://bitbucket.org/cffi/cffi/changeset/e7063ce4bdf8/

Log:	including <complex.h> fails on some systems with C++

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2010,7 +2010,7 @@
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
         static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2024,7 +2024,7 @@
     lib = verify(ffi, "test_function_returns_double_complex", """
         #include <complex.h>
         static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2038,7 +2038,7 @@
     lib = verify(ffi, "test_function_argument_float_complex", """
         #include <complex.h>
         static float f1(float _Complex x) { return cabsf(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-5
@@ -2051,7 +2051,7 @@
     lib = verify(ffi, "test_function_argument_double_complex", """
         #include <complex.h>
         static double f1(double _Complex x) { return cabs(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-11

From pypy.commits at gmail.com  Sun Jun  4 17:44:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 14:44:25 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: import cffi/e7063ce4bdf8
Message-ID: <59347f39.10d91c0a.31366.47d6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91516:35878bce885d
Date: 2017-06-04 23:43 +0200
http://bitbucket.org/pypy/pypy/changeset/35878bce885d/

Log:	import cffi/e7063ce4bdf8

diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2011,7 +2011,7 @@
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
         static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2025,7 +2025,7 @@
     lib = verify(ffi, "test_function_returns_double_complex", """
         #include <complex.h>
         static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2039,7 +2039,7 @@
     lib = verify(ffi, "test_function_argument_float_complex", """
         #include <complex.h>
         static float f1(float _Complex x) { return cabsf(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-5
@@ -2052,7 +2052,7 @@
     lib = verify(ffi, "test_function_argument_double_complex", """
         #include <complex.h>
         static double f1(double _Complex x) { return cabs(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-11

From pypy.commits at gmail.com  Mon Jun  5 02:25:20 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 23:25:20 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge cffi-char16-char32
Message-ID: <5934f950.aba0df0a.5adc2.642e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91518:6a4af0b6b51c
Date: 2017-06-05 08:24 +0200
http://bitbucket.org/pypy/pypy/changeset/6a4af0b6b51c/

Log:	hg merge cffi-char16-char32

	Support the char16_t and char32_t types in cffi. This means
	reintroducing some surrogate handling in one of the two directions,
	depending on the size of unichar.

diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -159,9 +159,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +174,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +219,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,5 +6,6 @@
 .. startrev: 558bd00b3dd8
 
 .. branch: cffi-complex
+.. branch: cffi-char16-char32
 
-Part of the upgrade to cffi 1.11
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,7 +9,7 @@
 
 
 VERSION_MIN    = 0x2601
-VERSION_MAX    = 0x27FF
+VERSION_MAX    = 0x28FF
 
 VERSION_EXPORT = 0x0A03
 
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -107,8 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +133,12 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -36,8 +36,7 @@
         datasize = self.size
         #
         if datasize < 0:
-            from pypy.module._cffi_backend import misc
-            w_init, length = misc.get_new_array_length(space, w_init)
+            w_init, length = self.get_new_array_length(w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:
@@ -53,6 +52,29 @@
                 self.convert_from_object(ptr, w_init)
         return cdata
 
+    def get_new_array_length(self, w_value):
+        space = self.space
+        if (space.isinstance_w(w_value, space.w_list) or
+            space.isinstance_w(w_value, space.w_tuple)):
+            return (w_value, space.int_w(space.len(w_value)))
+        elif space.isinstance_w(w_value, space.w_bytes):
+            # from a string, we add the null terminator
+            s = space.bytes_w(w_value)
+            return (w_value, len(s) + 1)
+        elif space.isinstance_w(w_value, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_value)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            return (w_value, length + 1)
+        else:
+            explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+            if explicitlength < 0:
+                raise oefmt(space.w_ValueError, "negative array length")
+            return (space.w_None, explicitlength)
+
     def _check_subscript_index(self, w_cdata, i):
         space = self.space
         if i < 0:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.tool import rfficache
 
 from pypy.interpreter.error import oefmt
-from pypy.module._cffi_backend import cdataobj, misc
+from pypy.module._cffi_backend import cdataobj, misc, wchar_helper
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
@@ -42,11 +42,13 @@
     def cast_unicode(self, w_ob):
         space = self.space
         s = space.unicode_w(w_ob)
-        if len(s) != 1:
+        try:
+            ordinal = wchar_helper.unicode_to_ordinal(s)
+        except ValueError:
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return intmask(ordinal)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
@@ -148,53 +150,83 @@
         return self.space.newbytes(s)
 
 
-# XXX explicitly use an integer type instead of lltype.UniChar here,
-# because for now the latter is defined as unsigned by RPython (even
-# though it may be signed when 'wchar_t' is written to C).
-WCHAR_INT = {(2, False): rffi.USHORT,
-             (4, False): rffi.UINT,
-             (4, True): rffi.INT}[rffi.sizeof(lltype.UniChar),
-                                  rfficache.signof_c_type('wchar_t')]
-WCHAR_INTP = rffi.CArrayPtr(WCHAR_INT)
+class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
+    _attrs_            = ['is_signed_wchar']
+    _immutable_fields_ = ['is_signed_wchar']
 
-class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
-    _attrs_ = []
+    _wchar_is_signed = rfficache.signof_c_type('wchar_t')
 
-    if rffi.r_wchar_t.SIGN:
-        def write_raw_integer_data(self, w_cdata, value):
-            w_cdata.write_raw_signed_data(value)
+    def __init__(self, space, size, name, name_position, align):
+        W_CTypePrimitiveCharOrUniChar.__init__(self, space, size, name,
+                                               name_position, align)
+        self.is_signed_wchar = self._wchar_is_signed and (name == "wchar_t")
+        # "char16_t" and "char32_t" are always unsigned
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.newint(unichardata[0])
+        if self.is_signed_wchar:
+            value = misc.read_raw_long_data(cdata, self.size)
+            return self.space.newint(value)
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)
+            if self.size < rffi.sizeof(lltype.Signed):
+                return self.space.newint(intmask(value))
+            else:
+                return self.space.newint(value)    # r_uint => 'long' object
 
     def convert_to_object(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.newunicode(unichardata[0])
+        if self.is_signed_wchar:
+            unichardata = rffi.cast(rffi.CWCHARP, cdata)
+            return self.space.newunicode(unichardata[0])
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)   # r_uint
+            try:
+                u = wchar_helper.ordinal_to_unicode(value)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
+            return self.space.newunicode(u)
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
             w_res = self.convert_to_object(ptr)
         return w_res
 
-    def _convert_to_unichar(self, w_ob):
+    def _convert_to_charN_t(self, w_ob):
+        # returns a r_uint.  If self.size == 2, it is smaller than 0x10000
         space = self.space
         if space.isinstance_w(w_ob, space.w_unicode):
-            s = space.unicode_w(w_ob)
-            if len(s) == 1:
-                return s[0]
-        if (isinstance(w_ob, cdataobj.W_CData) and
-               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
+            u = space.unicode_w(w_ob)
+            try:
+                ordinal = wchar_helper.unicode_to_ordinal(u)
+            except ValueError:
+                pass
+            else:
+                if self.size == 2 and ordinal > 0xffff:
+                    raise self._convert_error("single character <= 0xFFFF",
+                                              w_ob)
+                return ordinal
+        elif (isinstance(w_ob, cdataobj.W_CData) and
+               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
+               w_ob.ctype.size == self.size):
             with w_ob as ptr:
-                return rffi.cast(rffi.CWCHARP, ptr)[0]
+                return misc.read_raw_ulong_data(ptr, self.size)
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
-        value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        ordinal = self._convert_to_charN_t(w_ob)
+        misc.write_raw_unsigned_data(cdata, ordinal, self.size)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
-        u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+        if self.size == 2:
+            u = wchar_helper.unicode_from_char16(ptr, length)
+        else:
+            try:
+                u = wchar_helper.unicode_from_char32(ptr, length)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
         return self.space.newunicode(u)
 
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -4,9 +4,9 @@
 
 from rpython.rlib import rposix
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rtyper.annlowlevel import llstr, llunicode
+from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
@@ -88,18 +88,28 @@
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
+            from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
                 raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
-            n = len(s)
+            if self.ctitem.size == 2:
+                n = wchar_helper.unicode_size_as_char16(s)
+            else:
+                n = wchar_helper.unicode_size_as_char32(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
-            if n != self.length:
-                unichardata[n] = u'\x00'
+            add_final_zero = (n != self.length)
+            if self.ctitem.size == 2:
+                try:
+                    wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+                except wchar_helper.OutOfRange as e:
+                    raise oefmt(self.space.w_ValueError,
+                                "unicode character ouf of range for "
+                                "conversion to char16_t: %s", hex(e.ordinal))
+            else:
+                wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
@@ -134,12 +144,12 @@
                 #
                 # pointer to a wchar_t: builds and returns a unicode
                 if self.is_unichar_ptr_or_array():
-                    cdata = rffi.cast(rffi.CWCHARP, ptr)
-                    if length < 0:
-                        u = rffi.wcharp2unicode(cdata)
+                    from pypy.module._cffi_backend import wchar_helper
+                    if self.ctitem.size == 2:
+                        length = wchar_helper.measure_length_16(ptr, length)
                     else:
-                        u = rffi.wcharp2unicoden(cdata, length)
-                    return space.newunicode(u)
+                        length = wchar_helper.measure_length_32(ptr, length)
+                    return self.ctitem.unpack_ptr(self, ptr, length)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -302,9 +312,18 @@
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
-        elif space.isinstance_w(w_init, space.w_basestring):
+        elif space.isinstance_w(w_init, space.w_bytes):
             # from a string, we add the null terminator
-            length = space.int_w(space.len(w_init)) + 1
+            s = space.bytes_w(w_init)
+            length = len(s) + 1
+        elif space.isinstance_w(w_init, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_init)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            length += 1
         elif self.is_file:
             result = self.prepare_file(w_init)
             if result:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -244,7 +244,7 @@
         ct = self.ctype
         if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
             space = ct.space
-            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            w_ob, varsizelength = ct.get_new_array_length(w_ob)
             if optvarsize != -1:
                 # in this mode, the only purpose of this function is to compute
                 # the real size of the structure from a var-sized C99 array
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -290,21 +290,6 @@
 
 # ____________________________________________________________
 
-def get_new_array_length(space, w_value):
-    if (space.isinstance_w(w_value, space.w_list) or
-        space.isinstance_w(w_value, space.w_tuple)):
-        return (w_value, space.int_w(space.len(w_value)))
-    elif space.isinstance_w(w_value, space.w_basestring):
-        # from a string, we add the null terminator
-        return (w_value, space.int_w(space.len(w_value)) + 1)
-    else:
-        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
-        if explicitlength < 0:
-            raise oefmt(space.w_ValueError, "negative array length")
-        return (space.w_None, explicitlength)
-
-# ____________________________________________________________
-
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -111,6 +111,9 @@
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+eptypesize("char16_t", 2, ctypeprim.W_CTypePrimitiveUniChar)
+eptypesize("char32_t", 4, ctypeprim.W_CTypePrimitiveUniChar)
+
 _WCTSigned = ctypeprim.W_CTypePrimitiveSigned
 _WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
 
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -73,6 +73,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -505,6 +505,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -513,6 +514,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -80,8 +80,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1925,7 +1925,11 @@
         assert string(a, 8).startswith(b'ABC')  # may contain additional garbage
 
 def test_string_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     assert string(cast(BWChar, 42)) == u+'*'
     assert string(cast(BWChar, 0x4253)) == u+'\u4253'
     assert string(cast(BWChar, 0)) == u+'\x00'
@@ -2087,22 +2091,44 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+    if sys.platform.startswith("linux"):
+        BWChar = new_primitive_type("wchar_t")
+        assert sizeof(BWChar) == 4
+        assert int(cast(BWChar, -1)) == -1        # signed, on linux
+
+def test_char16():
+    BChar16 = new_primitive_type("char16_t")
+    assert sizeof(BChar16) == 2
+    _test_wchar_variant("char16_t")
+    assert int(cast(BChar16, -1)) == 0xffff       # always unsigned
+
+def test_char32():
+    BChar32 = new_primitive_type("char32_t")
+    assert sizeof(BChar32) == 4
+    _test_wchar_variant("char32_t")
+    assert int(cast(BChar32, -1)) == 0xffffffff   # always unsigned
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2117,9 +2143,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2154,17 +2180,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2172,8 +2198,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2200,7 +2226,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)
@@ -2213,6 +2239,27 @@
         x = cast(BWChar, -1)
         py.test.raises(ValueError, string, x)
 
+def test_wchar_variants_mix():
+    BWChar  = new_primitive_type("wchar_t")
+    BChar16 = new_primitive_type("char16_t")
+    BChar32 = new_primitive_type("char32_t")
+    assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+    assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+    #
+    BChar16A = new_array_type(new_pointer_type(BChar16), None)
+    BChar32A = new_array_type(new_pointer_type(BChar32), None)
+    x = cast(BChar32, 'A')
+    py.test.raises(TypeError, newp, BChar16A, [x])
+    x = cast(BChar16, 'A')
+    py.test.raises(TypeError, newp, BChar32A, [x])
+    #
+    a = newp(BChar16A, u+'\U00012345')
+    assert len(a) == 3
+    a = newp(BChar32A, u+'\U00012345')
+    assert len(a) == 2   # even if the Python unicode string above is 2 chars
+
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
@@ -3439,14 +3486,15 @@
     py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
     py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
     #
-    BUniChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BUniChar), None)
-    p = newp(BArray, u+"foobar")
-    p[2:5] = [u+"*", u+"Z", u+"T"]
-    p[1:3] = u+"XY"
-    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
-    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
-    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BUniChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BUniChar), None)
+        p = newp(BArray, u+"foobar")
+        p[2:5] = [u+"*", u+"Z", u+"T"]
+        p[1:3] = u+"XY"
+        assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+        py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+        py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
 
 def test_void_p_arithmetic():
     BVoid = new_void_type()
@@ -3759,10 +3807,12 @@
     p0 = p
     assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
     assert unpack(p+1, 5) == b"bc\x00de"
-    BWChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
-    p = newp(BArray, u"abc\x00def")
-    assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BWChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
+        p = newp(BArray, u"abc\x00def")
+        assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
 
     for typename, samples in [
             ("uint8_t",  [0, 2**8-1]),
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -555,3 +555,11 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         raises(ffi.error, ffi.cast, "int[-5]", 0)
+
+    def test_char32_t(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        z = ffi.new("char32_t[]", u'\U00012345')
+        assert len(z) == 2
+        assert ffi.cast("int *", z)[0] == 0x12345
+        assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -0,0 +1,192 @@
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.annlowlevel import llunicode
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_unicode_to_raw
+
+SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
+
+
+if SIZE_UNICODE == 4:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        return unichr(intmask(ordinal))
+else:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        if ordinal <= 0xffff:
+            return unichr(intmask(ordinal))
+        elif ordinal <= 0x10ffff:
+            ordinal = intmask(ordinal - 0x10000)
+            return (unichr(0xD800 | (ordinal >> 10)) +
+                    unichr(0xDC00 | (ordinal & 0x3FF)))
+        else:
+            raise OutOfRange(ordinal)
+
+def is_surrogate(u, index):
+    return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+            unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
+
+def as_surrogate(u, index):
+    ordinal = (ord(u[index + 0]) - 0xD800) << 10
+    ordinal |= (ord(u[index + 1]) - 0xDC00)
+    return r_uint(ordinal + 0x10000)
+
+def unicode_to_ordinal(u):
+    if len(u) == 1:
+        u = ord(u[0])
+        return r_uint(u)
+    elif SIZE_UNICODE == 2:
+        if len(u) == 2 and is_surrogate(u, 0):
+            return r_uint(as_surrogate(u, 0))
+    raise ValueError
+
+
+class OutOfRange(Exception):
+    ordinal = 0
+
+    def __init__(self, ordinal):
+        ordinal = intmask(rffi.cast(rffi.INT, ordinal))
+        self.ordinal = ordinal
+
+def _unicode_from_wchar(ptr, length):
+    return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+
+if SIZE_UNICODE == 2:
+    def unicode_from_char32(ptr, length):
+        # 'ptr' is a pointer to 'length' 32-bit integers
+        ptr = rffi.cast(rffi.UINTP, ptr)
+        alloc = length
+        for i in range(length):
+            if rffi.cast(lltype.Unsigned, ptr[i]) > 0xFFFF:
+                alloc += 1
+
+        u = [u'\x00'] * alloc
+        j = 0
+        for i in range(length):
+            ordinal = rffi.cast(lltype.Unsigned, ptr[i])
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal = intmask(ordinal - 0x10000)
+                u[j] = unichr(0xD800 | (ordinal >> 10))
+                j += 1
+                u[j] = unichr(0xDC00 | (ordinal & 0x3FF))
+                j += 1
+            else:
+                u[j] = unichr(intmask(ordinal))
+                j += 1
+        assert j == len(u)
+        return u''.join(u)
+
+    unicode_from_char16 = _unicode_from_wchar
+
+else:
+    unicode_from_char32 = _unicode_from_wchar
+
+    def unicode_from_char16(ptr, length):
+        # 'ptr' is a pointer to 'length' 16-bit integers
+        ptr = rffi.cast(rffi.USHORTP, ptr)
+        u = [u'\x00'] * length
+        i = 0
+        j = 0
+        while j < length:
+            ch = intmask(ptr[j])
+            j += 1
+            if 0xD800 <= ch <= 0xDBFF and j < length:
+                ch2 = intmask(ptr[j])
+                if 0xDC00 <= ch2 <= 0xDFFF:
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000
+                    j += 1
+            u[i] = unichr(ch)
+            i += 1
+        del u[i:]
+        return u''.join(u)
+
+
+ at specialize.ll()
+def _measure_length(ptr, maxlen):
+    result = 0
+    if maxlen < 0:
+        while intmask(ptr[result]) != 0:
+            result += 1
+    else:
+        while result < maxlen and intmask(ptr[result]) != 0:
+            result += 1
+    return result
+
+def measure_length_16(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.USHORTP, ptr), maxlen)
+
+def measure_length_32(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
+
+
+def unicode_size_as_char16(u):
+    result = len(u)
+    if SIZE_UNICODE == 4:
+        for i in range(result):
+            if ord(u[i]) > 0xFFFF:
+                result += 1
+    return result
+
+def unicode_size_as_char32(u):
+    result = len(u)
+    if SIZE_UNICODE == 2 and result > 1:
+        for i in range(result - 1):
+            if is_surrogate(u, i):
+                result -= 1
+    return result
+
+
+def _unicode_to_wchar(u, target_ptr, target_length, add_final_zero):
+    # 'target_ptr' is a raw pointer to 'target_length' wchars;
+    # we assume here that target_length == len(u).
+    unichardata = rffi.cast(rffi.CWCHARP, target_ptr)
+    copy_unicode_to_raw(llunicode(u), unichardata, 0, target_length)
+    if add_final_zero:
+        unichardata[target_length] = u'\x00'
+
+
+if SIZE_UNICODE == 2:
+    def unicode_to_char32(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 32-bit integers;
+        # we assume here that target_length == unicode_size_as_char32(u).
+        ptr = rffi.cast(rffi.UINTP, target_ptr)
+        src_index = 0
+        last_surrogate_pos = len(u) - 2
+        for i in range(target_length):
+            if src_index <= last_surrogate_pos and is_surrogate(u, src_index):
+                ordinal = as_surrogate(u, src_index)
+                src_index += 2
+            else:
+                ordinal = r_uint(ord(u[src_index]))
+                src_index += 1
+            ptr[i] = rffi.cast(rffi.UINT, ordinal)
+        if add_final_zero:
+            ptr[target_length] = rffi.cast(rffi.UINT, 0)
+
+    unicode_to_char16 = _unicode_to_wchar
+
+else:
+    unicode_to_char32 = _unicode_to_wchar
+
+    def unicode_to_char16(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 16-bit integers;
+        # we assume here that target_length == unicode_size_as_char16(u).
+        ptr = rffi.cast(rffi.USHORTP, target_ptr)
+        for uc in u:
+            ordinal = ord(uc)
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal -= 0x10000
+                ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
+                ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))
+                ptr = rffi.ptradd(ptr, 2)
+            else:
+                ptr[0] = rffi.cast(rffi.USHORT, ordinal)
+                ptr = rffi.ptradd(ptr, 1)
+        assert ptr == (
+            rffi.ptradd(rffi.cast(rffi.USHORTP, target_ptr), target_length))
+        if add_final_zero:
+            ptr[0] = rffi.cast(rffi.USHORT, 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -2,6 +2,7 @@
 import py, sys, platform
 import pytest
 from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 from cffi import FFI
 import _cffi_backend
 
@@ -398,6 +399,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -509,3 +512,43 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+    def test_explicitly_defined_char16_t(self):
+        ffi = FFI()
+        ffi.cdef("typedef uint16_t char16_t;")
+        x = ffi.cast("char16_t", 1234)
+        assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+    def test_char16_t(self):
+        ffi = FFI()
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        ffi = FFI()
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        py_uni = u+'\U00012345'
+        z = ffi.new("char32_t[]", py_uni)
+        assert len(z) == 2
+        assert list(z) == [py_uni, u+'\x00']    # maybe a 2-unichars string
+        assert ffi.string(z) == py_uni
+        if len(py_uni) == 1:    # 4-bytes unicodes in Python
+            s = ffi.new("char32_t[]", u+'\ud808\udf00')
+            assert len(s) == 3
+            assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -3,6 +3,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 
 
 SOURCE = """\
@@ -93,6 +94,15 @@
 }
 
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+    return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+    return (unsigned int)(a + 42);
+}
 """
 
 class TestOwnLib(object):
@@ -301,3 +311,18 @@
         pfn = ffi.addressof(lib, "test_getting_errno")
         assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
         assert pfn == lib.test_getting_errno
+
+    def test_char16_char32_t(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            char16_t foo_2bytes(char16_t);
+            char32_t foo_4bytes(char32_t);
+        """)
+        lib = ffi.dlopen(self.module)
+        assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -242,7 +242,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true
@@ -385,6 +385,10 @@
     lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
     assert lib.foo(uniexample1) == uniexample2
 
+def test_char16_char32_type():
+    py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+                 "working in ffi.verify() mode")
+
 def test_no_argument():
     ffi = FFI()
     ffi.cdef("int foo(void);")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1673,6 +1673,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -1743,3 +1745,30 @@
         exec("from _test_import_from_lib import *", d)
         assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
                 ['ffi', 'lib'])
+
+    def test_char16_t(self):
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,13 +25,14 @@
     assert ''.join(map(str, recomp.cffi_types)) == expected_output
 
 def verify(ffi, module_name, source, *args, **kwds):
+    no_cpp = kwds.pop('no_cpp', False)
     kwds.setdefault('undef_macros', ['NDEBUG'])
     module_name = '_CFFI_' + module_name
     ffi.set_source(module_name, source)
-    if not os.environ.get('NO_CPP'):     # test the .cpp mode too
+    if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
-    else:
+    elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
@@ -2010,7 +2011,7 @@
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
         static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2024,7 +2025,7 @@
     lib = verify(ffi, "test_function_returns_double_complex", """
         #include <complex.h>
         static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2038,7 +2039,7 @@
     lib = verify(ffi, "test_function_argument_float_complex", """
         #include <complex.h>
         static float f1(float _Complex x) { return cabsf(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-5
@@ -2051,7 +2052,7 @@
     lib = verify(ffi, "test_function_argument_double_complex", """
         #include <complex.h>
         static double f1(double _Complex x) { return cabs(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-11
@@ -2251,3 +2252,34 @@
     int f(int a) { return a + 40; }
     """, extra_compile_args=['-fvisibility=hidden'])
     assert lib.f(2) == 42
+
+def test_override_default_definition():
+    ffi = FFI()
+    ffi.cdef("typedef long int16_t, char16_t;")
+    lib = verify(ffi, "test_override_default_definition", "")
+    assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+    ffi = FFI()
+    ffi.cdef("""
+        char16_t foo_2bytes(char16_t);
+        char32_t foo_4bytes(char32_t);
+    """)
+    lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    #if !defined(__cplusplus) || __cplusplus < 201103L
+    typedef uint_least16_t char16_t;
+    typedef uint_least32_t char32_t;
+    #endif
+    
+    char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+    char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+    """, no_cpp=no_cpp)
+    assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+    py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+    py.test.raises(TypeError, lib.foo_2bytes, 1234)
+    py.test.raises(TypeError, lib.foo_4bytes, 1234)
+
+def test_char16_char32_plain_c():
+    test_char16_char32_type(no_cpp=True)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -222,7 +222,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true

From pypy.commits at gmail.com  Mon Jun  5 02:25:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 04 Jun 2017 23:25:18 -0700 (PDT)
Subject: [pypy-commit] pypy cffi-char16-char32: ready to merge
Message-ID: <5934f94e.11addf0a.2b75b.5c76@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: cffi-char16-char32
Changeset: r91517:0f1cd402bb00
Date: 2017-06-05 08:23 +0200
http://bitbucket.org/pypy/pypy/changeset/0f1cd402bb00/

Log:	ready to merge

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,5 +6,6 @@
 .. startrev: 558bd00b3dd8
 
 .. branch: cffi-complex
+.. branch: cffi-char16-char32
 
-Part of the upgrade to cffi 1.11
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.

From pypy.commits at gmail.com  Mon Jun  5 05:51:21 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 02:51:21 -0700 (PDT)
Subject: [pypy-commit] pypy default: Fix: 'tp_descr_get(self, NULL,
 type)' used to give a real undefined
Message-ID: <59352999.44d21c0a.b3268.25c2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91519:07257b27db4b
Date: 2017-06-05 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/07257b27db4b/

Log:	Fix: 'tp_descr_get(self, NULL, type)' used to give a real undefined
	value for the second argument if implemented in Python

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -51,7 +51,7 @@
         w_descr = space.appexec([], """():
             class Descr(object):
                 def __get__(self, obj, type):
-                    return 42
+                    return 42 + (obj is None)
                 def __set__(self, obj, value):
                     obj.append('set')
                 def __delete__(self, obj):
@@ -73,6 +73,11 @@
             space, py_descrtype.c_tp_descr_set,
             py_descr, py_obj, None) == 0
         assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+        #
+        # unbound __get__(self, NULL, type)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, None, space.w_int)
+        assert space.int_w(w_res) == 43
 
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -111,6 +111,8 @@
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    if w_obj is None:
+        w_obj = space.w_None
     return space.get(w_self, w_obj, w_type)
 
 @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)

From pypy.commits at gmail.com  Mon Jun  5 05:51:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 02:51:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: tp_descr_get on built-in types
Message-ID: <5935299b.15371c0a.42f9f.b58c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91520:4a138a88b24a
Date: 2017-06-05 11:26 +0200
http://bitbucket.org/pypy/pypy/changeset/4a138a88b24a/

Log:	tp_descr_get on built-in types

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -820,6 +820,18 @@
         else:
             return
         slot_func = buff_w
+    elif name == 'tp_descr_get':
+        get_fn = w_type.getdictvalue(space, '__get__')
+        if get_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
+            if w_arg1 is None:
+                w_arg1 = space.w_None
+            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        slot_func = slot_tp_descr_get
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -392,6 +392,37 @@
         obj = foo.new()
         assert module.hack_tp_dict(obj) == 2
 
+    def test_tp_descr_get(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_get", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_get == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                return args->ob_type->tp_descr_get(args, NULL,
+                                                   (PyObject *)&PyInt_Type);
+             '''
+             )
+            ])
+        assert module.tp_descr_get(42) is False
+
+        class Y(object):
+            def __get__(self, *args):
+                return 42
+            def unbound_method_example(self):
+                pass
+        assert module.tp_descr_get(Y()) == 42
+        #
+        p = property(lambda self: 42)
+        result = module.tp_descr_get(p)
+        assert result is p
+        #
+        f = lambda x: x + 1
+        ubm = module.tp_descr_get(f)
+        assert type(ubm) is type(Y.unbound_method_example)
+        assert ubm(42) == 43
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 05:51:25 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 02:51:25 -0700 (PDT)
Subject: [pypy-commit] pypy default: tp_descr_set on built-in types
Message-ID: <5935299d.05421c0a.4cd7a.43be@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91521:7ca42aeab616
Date: 2017-06-05 11:42 +0200
http://bitbucket.org/pypy/pypy/changeset/7ca42aeab616/

Log:	tp_descr_set on built-in types

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -827,11 +827,32 @@
 
         @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
-        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
-            if w_arg1 is None:
-                w_arg1 = space.w_None
-            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        def slot_tp_descr_get(space, w_self, w_obj, w_value):
+            if w_obj is None:
+                w_obj = space.w_None
+            return space.call_function(get_fn, w_self, w_obj, w_value)
         slot_func = slot_tp_descr_get
+    elif name == 'tp_descr_set':
+        set_fn = w_type.getdictvalue(space, '__set__')
+        delete_fn = w_type.getdictvalue(space, '__delete__')
+        if set_fn is None and delete_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_set(space, w_self, w_obj, w_value):
+            if w_value is not None:
+                if set_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __set__", typedef.name)
+                space.call_function(set_fn, w_self, w_obj, w_value)
+            else:
+                if delete_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __delete__", typedef.name)
+                space.call_function(delete_fn, w_self, w_obj)
+            return 0
+        slot_func = slot_tp_descr_set
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -423,6 +423,42 @@
         assert type(ubm) is type(Y.unbound_method_example)
         assert ubm(42) == 43
 
+    def test_tp_descr_set(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_set", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_set == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                if (args->ob_type->tp_descr_set(args, Py_False, Py_True) != 0)
+                    return NULL;
+                if (args->ob_type->tp_descr_set(args, Py_Ellipsis, NULL) != 0)
+                    return NULL;
+
+                Py_INCREF(Py_True);
+                return Py_True;
+             '''
+             )
+            ])
+        assert module.tp_descr_set(42) is False
+
+        class Y(object):
+            def __set__(self, obj, value):
+                assert obj is False
+                assert value is True
+            def __delete__(self, obj):
+                assert obj is Ellipsis
+        assert module.tp_descr_set(Y()) is True
+        #
+        def pset(obj, value):
+            assert obj is False
+            assert value is True
+        def pdel(obj):
+            assert obj is Ellipsis
+        p = property(lambda: "never used", pset, pdel)
+        assert module.tp_descr_set(p) is True
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 05:51:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 02:51:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: hg merge heads
Message-ID: <5935299f.42131c0a.d618f.9cca@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91522:0649d557369f
Date: 2017-06-05 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/0649d557369f/

Log:	hg merge heads

diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
 
     def __init__(self, message, category, filename, lineno, file=None,
                     line=None):
-        local_values = locals()
-        for attr in self._WARNING_DETAILS:
-            setattr(self, attr, local_values[attr])
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
         self._category_name = category.__name__ if category else None
 
     def __str__(self):
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -159,9 +159,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +174,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +219,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,5 +6,6 @@
 .. startrev: 558bd00b3dd8
 
 .. branch: cffi-complex
+.. branch: cffi-char16-char32
 
-Part of the upgrade to cffi 1.11
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,7 +9,7 @@
 
 
 VERSION_MIN    = 0x2601
-VERSION_MAX    = 0x27FF
+VERSION_MAX    = 0x28FF
 
 VERSION_EXPORT = 0x0A03
 
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -107,8 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +133,12 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -36,8 +36,7 @@
         datasize = self.size
         #
         if datasize < 0:
-            from pypy.module._cffi_backend import misc
-            w_init, length = misc.get_new_array_length(space, w_init)
+            w_init, length = self.get_new_array_length(w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:
@@ -53,6 +52,29 @@
                 self.convert_from_object(ptr, w_init)
         return cdata
 
+    def get_new_array_length(self, w_value):
+        space = self.space
+        if (space.isinstance_w(w_value, space.w_list) or
+            space.isinstance_w(w_value, space.w_tuple)):
+            return (w_value, space.int_w(space.len(w_value)))
+        elif space.isinstance_w(w_value, space.w_bytes):
+            # from a string, we add the null terminator
+            s = space.bytes_w(w_value)
+            return (w_value, len(s) + 1)
+        elif space.isinstance_w(w_value, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_value)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            return (w_value, length + 1)
+        else:
+            explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+            if explicitlength < 0:
+                raise oefmt(space.w_ValueError, "negative array length")
+            return (space.w_None, explicitlength)
+
     def _check_subscript_index(self, w_cdata, i):
         space = self.space
         if i < 0:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.tool import rfficache
 
 from pypy.interpreter.error import oefmt
-from pypy.module._cffi_backend import cdataobj, misc
+from pypy.module._cffi_backend import cdataobj, misc, wchar_helper
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
@@ -42,11 +42,13 @@
     def cast_unicode(self, w_ob):
         space = self.space
         s = space.unicode_w(w_ob)
-        if len(s) != 1:
+        try:
+            ordinal = wchar_helper.unicode_to_ordinal(s)
+        except ValueError:
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return intmask(ordinal)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
@@ -148,53 +150,83 @@
         return self.space.newbytes(s)
 
 
-# XXX explicitly use an integer type instead of lltype.UniChar here,
-# because for now the latter is defined as unsigned by RPython (even
-# though it may be signed when 'wchar_t' is written to C).
-WCHAR_INT = {(2, False): rffi.USHORT,
-             (4, False): rffi.UINT,
-             (4, True): rffi.INT}[rffi.sizeof(lltype.UniChar),
-                                  rfficache.signof_c_type('wchar_t')]
-WCHAR_INTP = rffi.CArrayPtr(WCHAR_INT)
+class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
+    _attrs_            = ['is_signed_wchar']
+    _immutable_fields_ = ['is_signed_wchar']
 
-class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
-    _attrs_ = []
+    _wchar_is_signed = rfficache.signof_c_type('wchar_t')
 
-    if rffi.r_wchar_t.SIGN:
-        def write_raw_integer_data(self, w_cdata, value):
-            w_cdata.write_raw_signed_data(value)
+    def __init__(self, space, size, name, name_position, align):
+        W_CTypePrimitiveCharOrUniChar.__init__(self, space, size, name,
+                                               name_position, align)
+        self.is_signed_wchar = self._wchar_is_signed and (name == "wchar_t")
+        # "char16_t" and "char32_t" are always unsigned
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.newint(unichardata[0])
+        if self.is_signed_wchar:
+            value = misc.read_raw_long_data(cdata, self.size)
+            return self.space.newint(value)
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)
+            if self.size < rffi.sizeof(lltype.Signed):
+                return self.space.newint(intmask(value))
+            else:
+                return self.space.newint(value)    # r_uint => 'long' object
 
     def convert_to_object(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.newunicode(unichardata[0])
+        if self.is_signed_wchar:
+            unichardata = rffi.cast(rffi.CWCHARP, cdata)
+            return self.space.newunicode(unichardata[0])
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)   # r_uint
+            try:
+                u = wchar_helper.ordinal_to_unicode(value)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
+            return self.space.newunicode(u)
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
             w_res = self.convert_to_object(ptr)
         return w_res
 
-    def _convert_to_unichar(self, w_ob):
+    def _convert_to_charN_t(self, w_ob):
+        # returns a r_uint.  If self.size == 2, it is smaller than 0x10000
         space = self.space
         if space.isinstance_w(w_ob, space.w_unicode):
-            s = space.unicode_w(w_ob)
-            if len(s) == 1:
-                return s[0]
-        if (isinstance(w_ob, cdataobj.W_CData) and
-               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
+            u = space.unicode_w(w_ob)
+            try:
+                ordinal = wchar_helper.unicode_to_ordinal(u)
+            except ValueError:
+                pass
+            else:
+                if self.size == 2 and ordinal > 0xffff:
+                    raise self._convert_error("single character <= 0xFFFF",
+                                              w_ob)
+                return ordinal
+        elif (isinstance(w_ob, cdataobj.W_CData) and
+               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
+               w_ob.ctype.size == self.size):
             with w_ob as ptr:
-                return rffi.cast(rffi.CWCHARP, ptr)[0]
+                return misc.read_raw_ulong_data(ptr, self.size)
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
-        value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        ordinal = self._convert_to_charN_t(w_ob)
+        misc.write_raw_unsigned_data(cdata, ordinal, self.size)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
-        u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+        if self.size == 2:
+            u = wchar_helper.unicode_from_char16(ptr, length)
+        else:
+            try:
+                u = wchar_helper.unicode_from_char32(ptr, length)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
         return self.space.newunicode(u)
 
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -4,9 +4,9 @@
 
 from rpython.rlib import rposix
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rtyper.annlowlevel import llstr, llunicode
+from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
@@ -88,18 +88,28 @@
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
+            from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
                 raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
-            n = len(s)
+            if self.ctitem.size == 2:
+                n = wchar_helper.unicode_size_as_char16(s)
+            else:
+                n = wchar_helper.unicode_size_as_char32(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
-            if n != self.length:
-                unichardata[n] = u'\x00'
+            add_final_zero = (n != self.length)
+            if self.ctitem.size == 2:
+                try:
+                    wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+                except wchar_helper.OutOfRange as e:
+                    raise oefmt(self.space.w_ValueError,
+                                "unicode character ouf of range for "
+                                "conversion to char16_t: %s", hex(e.ordinal))
+            else:
+                wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
@@ -134,12 +144,12 @@
                 #
                 # pointer to a wchar_t: builds and returns a unicode
                 if self.is_unichar_ptr_or_array():
-                    cdata = rffi.cast(rffi.CWCHARP, ptr)
-                    if length < 0:
-                        u = rffi.wcharp2unicode(cdata)
+                    from pypy.module._cffi_backend import wchar_helper
+                    if self.ctitem.size == 2:
+                        length = wchar_helper.measure_length_16(ptr, length)
                     else:
-                        u = rffi.wcharp2unicoden(cdata, length)
-                    return space.newunicode(u)
+                        length = wchar_helper.measure_length_32(ptr, length)
+                    return self.ctitem.unpack_ptr(self, ptr, length)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -302,9 +312,18 @@
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
-        elif space.isinstance_w(w_init, space.w_basestring):
+        elif space.isinstance_w(w_init, space.w_bytes):
             # from a string, we add the null terminator
-            length = space.int_w(space.len(w_init)) + 1
+            s = space.bytes_w(w_init)
+            length = len(s) + 1
+        elif space.isinstance_w(w_init, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_init)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            length += 1
         elif self.is_file:
             result = self.prepare_file(w_init)
             if result:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -244,7 +244,7 @@
         ct = self.ctype
         if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
             space = ct.space
-            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            w_ob, varsizelength = ct.get_new_array_length(w_ob)
             if optvarsize != -1:
                 # in this mode, the only purpose of this function is to compute
                 # the real size of the structure from a var-sized C99 array
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -290,21 +290,6 @@
 
 # ____________________________________________________________
 
-def get_new_array_length(space, w_value):
-    if (space.isinstance_w(w_value, space.w_list) or
-        space.isinstance_w(w_value, space.w_tuple)):
-        return (w_value, space.int_w(space.len(w_value)))
-    elif space.isinstance_w(w_value, space.w_basestring):
-        # from a string, we add the null terminator
-        return (w_value, space.int_w(space.len(w_value)) + 1)
-    else:
-        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
-        if explicitlength < 0:
-            raise oefmt(space.w_ValueError, "negative array length")
-        return (space.w_None, explicitlength)
-
-# ____________________________________________________________
-
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -111,6 +111,9 @@
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+eptypesize("char16_t", 2, ctypeprim.W_CTypePrimitiveUniChar)
+eptypesize("char32_t", 4, ctypeprim.W_CTypePrimitiveUniChar)
+
 _WCTSigned = ctypeprim.W_CTypePrimitiveSigned
 _WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
 
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -73,6 +73,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -505,6 +505,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -513,6 +514,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -80,8 +80,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1925,7 +1925,11 @@
         assert string(a, 8).startswith(b'ABC')  # may contain additional garbage
 
 def test_string_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     assert string(cast(BWChar, 42)) == u+'*'
     assert string(cast(BWChar, 0x4253)) == u+'\u4253'
     assert string(cast(BWChar, 0)) == u+'\x00'
@@ -2087,22 +2091,44 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+    if sys.platform.startswith("linux"):
+        BWChar = new_primitive_type("wchar_t")
+        assert sizeof(BWChar) == 4
+        assert int(cast(BWChar, -1)) == -1        # signed, on linux
+
+def test_char16():
+    BChar16 = new_primitive_type("char16_t")
+    assert sizeof(BChar16) == 2
+    _test_wchar_variant("char16_t")
+    assert int(cast(BChar16, -1)) == 0xffff       # always unsigned
+
+def test_char32():
+    BChar32 = new_primitive_type("char32_t")
+    assert sizeof(BChar32) == 4
+    _test_wchar_variant("char32_t")
+    assert int(cast(BChar32, -1)) == 0xffffffff   # always unsigned
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2117,9 +2143,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2154,17 +2180,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2172,8 +2198,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2200,7 +2226,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)
@@ -2213,6 +2239,27 @@
         x = cast(BWChar, -1)
         py.test.raises(ValueError, string, x)
 
+def test_wchar_variants_mix():
+    BWChar  = new_primitive_type("wchar_t")
+    BChar16 = new_primitive_type("char16_t")
+    BChar32 = new_primitive_type("char32_t")
+    assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+    assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+    #
+    BChar16A = new_array_type(new_pointer_type(BChar16), None)
+    BChar32A = new_array_type(new_pointer_type(BChar32), None)
+    x = cast(BChar32, 'A')
+    py.test.raises(TypeError, newp, BChar16A, [x])
+    x = cast(BChar16, 'A')
+    py.test.raises(TypeError, newp, BChar32A, [x])
+    #
+    a = newp(BChar16A, u+'\U00012345')
+    assert len(a) == 3
+    a = newp(BChar32A, u+'\U00012345')
+    assert len(a) == 2   # even if the Python unicode string above is 2 chars
+
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
@@ -3439,14 +3486,15 @@
     py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
     py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
     #
-    BUniChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BUniChar), None)
-    p = newp(BArray, u+"foobar")
-    p[2:5] = [u+"*", u+"Z", u+"T"]
-    p[1:3] = u+"XY"
-    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
-    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
-    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BUniChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BUniChar), None)
+        p = newp(BArray, u+"foobar")
+        p[2:5] = [u+"*", u+"Z", u+"T"]
+        p[1:3] = u+"XY"
+        assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+        py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+        py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
 
 def test_void_p_arithmetic():
     BVoid = new_void_type()
@@ -3759,10 +3807,12 @@
     p0 = p
     assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
     assert unpack(p+1, 5) == b"bc\x00de"
-    BWChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
-    p = newp(BArray, u"abc\x00def")
-    assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BWChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
+        p = newp(BArray, u"abc\x00def")
+        assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
 
     for typename, samples in [
             ("uint8_t",  [0, 2**8-1]),
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -555,3 +555,11 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         raises(ffi.error, ffi.cast, "int[-5]", 0)
+
+    def test_char32_t(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        z = ffi.new("char32_t[]", u'\U00012345')
+        assert len(z) == 2
+        assert ffi.cast("int *", z)[0] == 0x12345
+        assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -0,0 +1,192 @@
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.annlowlevel import llunicode
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_unicode_to_raw
+
+SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
+
+
+if SIZE_UNICODE == 4:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        return unichr(intmask(ordinal))
+else:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        if ordinal <= 0xffff:
+            return unichr(intmask(ordinal))
+        elif ordinal <= 0x10ffff:
+            ordinal = intmask(ordinal - 0x10000)
+            return (unichr(0xD800 | (ordinal >> 10)) +
+                    unichr(0xDC00 | (ordinal & 0x3FF)))
+        else:
+            raise OutOfRange(ordinal)
+
+def is_surrogate(u, index):
+    return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+            unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
+
+def as_surrogate(u, index):
+    ordinal = (ord(u[index + 0]) - 0xD800) << 10
+    ordinal |= (ord(u[index + 1]) - 0xDC00)
+    return r_uint(ordinal + 0x10000)
+
+def unicode_to_ordinal(u):
+    if len(u) == 1:
+        u = ord(u[0])
+        return r_uint(u)
+    elif SIZE_UNICODE == 2:
+        if len(u) == 2 and is_surrogate(u, 0):
+            return r_uint(as_surrogate(u, 0))
+    raise ValueError
+
+
+class OutOfRange(Exception):
+    ordinal = 0
+
+    def __init__(self, ordinal):
+        ordinal = intmask(rffi.cast(rffi.INT, ordinal))
+        self.ordinal = ordinal
+
+def _unicode_from_wchar(ptr, length):
+    return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+
+if SIZE_UNICODE == 2:
+    def unicode_from_char32(ptr, length):
+        # 'ptr' is a pointer to 'length' 32-bit integers
+        ptr = rffi.cast(rffi.UINTP, ptr)
+        alloc = length
+        for i in range(length):
+            if rffi.cast(lltype.Unsigned, ptr[i]) > 0xFFFF:
+                alloc += 1
+
+        u = [u'\x00'] * alloc
+        j = 0
+        for i in range(length):
+            ordinal = rffi.cast(lltype.Unsigned, ptr[i])
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal = intmask(ordinal - 0x10000)
+                u[j] = unichr(0xD800 | (ordinal >> 10))
+                j += 1
+                u[j] = unichr(0xDC00 | (ordinal & 0x3FF))
+                j += 1
+            else:
+                u[j] = unichr(intmask(ordinal))
+                j += 1
+        assert j == len(u)
+        return u''.join(u)
+
+    unicode_from_char16 = _unicode_from_wchar
+
+else:
+    unicode_from_char32 = _unicode_from_wchar
+
+    def unicode_from_char16(ptr, length):
+        # 'ptr' is a pointer to 'length' 16-bit integers
+        ptr = rffi.cast(rffi.USHORTP, ptr)
+        u = [u'\x00'] * length
+        i = 0
+        j = 0
+        while j < length:
+            ch = intmask(ptr[j])
+            j += 1
+            if 0xD800 <= ch <= 0xDBFF and j < length:
+                ch2 = intmask(ptr[j])
+                if 0xDC00 <= ch2 <= 0xDFFF:
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000
+                    j += 1
+            u[i] = unichr(ch)
+            i += 1
+        del u[i:]
+        return u''.join(u)
+
+
+ at specialize.ll()
+def _measure_length(ptr, maxlen):
+    result = 0
+    if maxlen < 0:
+        while intmask(ptr[result]) != 0:
+            result += 1
+    else:
+        while result < maxlen and intmask(ptr[result]) != 0:
+            result += 1
+    return result
+
+def measure_length_16(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.USHORTP, ptr), maxlen)
+
+def measure_length_32(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
+
+
+def unicode_size_as_char16(u):
+    result = len(u)
+    if SIZE_UNICODE == 4:
+        for i in range(result):
+            if ord(u[i]) > 0xFFFF:
+                result += 1
+    return result
+
+def unicode_size_as_char32(u):
+    result = len(u)
+    if SIZE_UNICODE == 2 and result > 1:
+        for i in range(result - 1):
+            if is_surrogate(u, i):
+                result -= 1
+    return result
+
+
+def _unicode_to_wchar(u, target_ptr, target_length, add_final_zero):
+    # 'target_ptr' is a raw pointer to 'target_length' wchars;
+    # we assume here that target_length == len(u).
+    unichardata = rffi.cast(rffi.CWCHARP, target_ptr)
+    copy_unicode_to_raw(llunicode(u), unichardata, 0, target_length)
+    if add_final_zero:
+        unichardata[target_length] = u'\x00'
+
+
+if SIZE_UNICODE == 2:
+    def unicode_to_char32(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 32-bit integers;
+        # we assume here that target_length == unicode_size_as_char32(u).
+        ptr = rffi.cast(rffi.UINTP, target_ptr)
+        src_index = 0
+        last_surrogate_pos = len(u) - 2
+        for i in range(target_length):
+            if src_index <= last_surrogate_pos and is_surrogate(u, src_index):
+                ordinal = as_surrogate(u, src_index)
+                src_index += 2
+            else:
+                ordinal = r_uint(ord(u[src_index]))
+                src_index += 1
+            ptr[i] = rffi.cast(rffi.UINT, ordinal)
+        if add_final_zero:
+            ptr[target_length] = rffi.cast(rffi.UINT, 0)
+
+    unicode_to_char16 = _unicode_to_wchar
+
+else:
+    unicode_to_char32 = _unicode_to_wchar
+
+    def unicode_to_char16(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 16-bit integers;
+        # we assume here that target_length == unicode_size_as_char16(u).
+        ptr = rffi.cast(rffi.USHORTP, target_ptr)
+        for uc in u:
+            ordinal = ord(uc)
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal -= 0x10000
+                ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
+                ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))
+                ptr = rffi.ptradd(ptr, 2)
+            else:
+                ptr[0] = rffi.cast(rffi.USHORT, ordinal)
+                ptr = rffi.ptradd(ptr, 1)
+        assert ptr == (
+            rffi.ptradd(rffi.cast(rffi.USHORTP, target_ptr), target_length))
+        if add_final_zero:
+            ptr[0] = rffi.cast(rffi.USHORT, 0)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py
@@ -2,6 +2,7 @@
 import py, sys, platform
 import pytest
 from pypy.module.test_lib_pypy.cffi_tests.cffi0 import backend_tests, test_function, test_ownlib
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 from cffi import FFI
 import _cffi_backend
 
@@ -398,6 +399,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -509,3 +512,43 @@
             py.test.raises(TypeError, cd)
             py.test.raises(TypeError, cd, ffi.NULL)
             py.test.raises(TypeError, cd, ffi.typeof("void *"))
+
+    def test_explicitly_defined_char16_t(self):
+        ffi = FFI()
+        ffi.cdef("typedef uint16_t char16_t;")
+        x = ffi.cast("char16_t", 1234)
+        assert ffi.typeof(x) is ffi.typeof("uint16_t")
+
+    def test_char16_t(self):
+        ffi = FFI()
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        ffi = FFI()
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        py_uni = u+'\U00012345'
+        z = ffi.new("char32_t[]", py_uni)
+        assert len(z) == 2
+        assert list(z) == [py_uni, u+'\x00']    # maybe a 2-unichars string
+        assert ffi.string(z) == py_uni
+        if len(py_uni) == 1:    # 4-bytes unicodes in Python
+            s = ffi.new("char32_t[]", u+'\ud808\udf00')
+            assert len(s) == 3
+            assert list(s) == [u+'\ud808', u+'\udf00', u+'\x00']
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ownlib.py
@@ -3,6 +3,7 @@
 import subprocess, weakref
 from cffi import FFI
 from cffi.backend_ctypes import CTypesBackend
+from pypy.module.test_lib_pypy.cffi_tests.support import u
 
 
 SOURCE = """\
@@ -93,6 +94,15 @@
 }
 
 EXPORT int my_array[7] = {0, 1, 2, 3, 4, 5, 6};
+
+EXPORT unsigned short foo_2bytes(unsigned short a)
+{
+    return (unsigned short)(a + 42);
+}
+EXPORT unsigned int foo_4bytes(unsigned int a)
+{
+    return (unsigned int)(a + 42);
+}
 """
 
 class TestOwnLib(object):
@@ -301,3 +311,18 @@
         pfn = ffi.addressof(lib, "test_getting_errno")
         assert ffi.typeof(pfn) == ffi.typeof("int(*)(void)")
         assert pfn == lib.test_getting_errno
+
+    def test_char16_char32_t(self):
+        if self.module is None:
+            py.test.skip("fix the auto-generation of the tiny test lib")
+        if self.Backend is CTypesBackend:
+            py.test.skip("not implemented with the ctypes backend")
+        ffi = FFI(backend=self.Backend())
+        ffi.cdef("""
+            char16_t foo_2bytes(char16_t);
+            char32_t foo_4bytes(char32_t);
+        """)
+        lib = ffi.dlopen(self.module)
+        assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+        assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py
@@ -242,7 +242,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true
@@ -385,6 +385,10 @@
     lib = ffi.verify("wchar_t foo(wchar_t x) { return x+1; }")
     assert lib.foo(uniexample1) == uniexample2
 
+def test_char16_char32_type():
+    py.test.skip("XXX test or fully prevent char16_t and char32_t from "
+                 "working in ffi.verify() mode")
+
 def test_no_argument():
     ffi = FFI()
     ffi.cdef("int foo(void);")
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -1673,6 +1673,8 @@
             "double",
             "long double",
             "wchar_t",
+            "char16_t",
+            "char32_t",
             "_Bool",
             "int8_t",
             "uint8_t",
@@ -1743,3 +1745,30 @@
         exec("from _test_import_from_lib import *", d)
         assert (sorted([x for x in d.keys() if not x.startswith('__')]) ==
                 ['ffi', 'lib'])
+
+    def test_char16_t(self):
+        x = ffi.new("char16_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 10
+        x[2] = u+'\u1324'
+        assert x[2] == u+'\u1324'
+        y = ffi.new("char16_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        assert ffi.string(y) == u+'\u1234\u5678'
+        z = ffi.new("char16_t[]", u+'\U00012345')
+        assert len(z) == 3
+        assert list(z) == [u+'\ud808', u+'\udf45', u+'\x00']
+        assert ffi.string(z) == u+'\U00012345'
+
+    def test_char32_t(self):
+        x = ffi.new("char32_t[]", 5)
+        assert len(x) == 5 and ffi.sizeof(x) == 20
+        x[3] = u+'\U00013245'
+        assert x[3] == u+'\U00013245'
+        y = ffi.new("char32_t[]", u+'\u1234\u5678')
+        assert len(y) == 3
+        assert list(y) == [u+'\u1234', u+'\u5678', u+'\x00']
+        z = ffi.new("char32_t[]", u+'\U00012345')
+        assert len(z) == 2
+        assert list(z) == [u+'\U00012345', u+'\x00'] # maybe a 2-unichars strin
+        assert ffi.string(z) == u+'\U00012345'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -25,13 +25,14 @@
     assert ''.join(map(str, recomp.cffi_types)) == expected_output
 
 def verify(ffi, module_name, source, *args, **kwds):
+    no_cpp = kwds.pop('no_cpp', False)
     kwds.setdefault('undef_macros', ['NDEBUG'])
     module_name = '_CFFI_' + module_name
     ffi.set_source(module_name, source)
-    if not os.environ.get('NO_CPP'):     # test the .cpp mode too
+    if not os.environ.get('NO_CPP') and not no_cpp:   # test the .cpp mode too
         kwds.setdefault('source_extension', '.cpp')
         source = 'extern "C" {\n%s\n}' % (source,)
-    else:
+    elif sys.platform != 'win32':
         # add '-Werror' to the existing 'extra_compile_args' flags
         kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) +
                                       ['-Werror'])
@@ -2010,7 +2011,7 @@
     lib = verify(ffi, "test_function_returns_float_complex", """
         #include <complex.h>
         static float _Complex f1(float a, float b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2024,7 +2025,7 @@
     lib = verify(ffi, "test_function_returns_double_complex", """
         #include <complex.h>
         static double _Complex f1(double a, double b) { return a + I*2.0*b; }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     result = lib.f1(1.25, 5.1)
     assert type(result) == complex
     assert result.real == 1.25   # exact
@@ -2038,7 +2039,7 @@
     lib = verify(ffi, "test_function_argument_float_complex", """
         #include <complex.h>
         static float f1(float _Complex x) { return cabsf(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-5
@@ -2051,7 +2052,7 @@
     lib = verify(ffi, "test_function_argument_double_complex", """
         #include <complex.h>
         static double f1(double _Complex x) { return cabs(x); }
-    """)
+    """, no_cpp=True)    # <complex.h> fails on some systems with C++
     x = complex(12.34, 56.78)
     result = lib.f1(x)
     assert abs(result - abs(x)) < 1e-11
@@ -2251,3 +2252,34 @@
     int f(int a) { return a + 40; }
     """, extra_compile_args=['-fvisibility=hidden'])
     assert lib.f(2) == 42
+
+def test_override_default_definition():
+    ffi = FFI()
+    ffi.cdef("typedef long int16_t, char16_t;")
+    lib = verify(ffi, "test_override_default_definition", "")
+    assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
+
+def test_char16_char32_type(no_cpp=False):
+    ffi = FFI()
+    ffi.cdef("""
+        char16_t foo_2bytes(char16_t);
+        char32_t foo_4bytes(char32_t);
+    """)
+    lib = verify(ffi, "test_char16_char32_type" + no_cpp * "_nocpp", """
+    #if !defined(__cplusplus) || __cplusplus < 201103L
+    typedef uint_least16_t char16_t;
+    typedef uint_least32_t char32_t;
+    #endif
+    
+    char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
+    char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
+    """, no_cpp=no_cpp)
+    assert lib.foo_2bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\u1234') == u+'\u125e'
+    assert lib.foo_4bytes(u+'\U00012345') == u+'\U0001236f'
+    py.test.raises(TypeError, lib.foo_2bytes, u+'\U00012345')
+    py.test.raises(TypeError, lib.foo_2bytes, 1234)
+    py.test.raises(TypeError, lib.foo_4bytes, 1234)
+
+def test_char16_char32_plain_c():
+    test_char16_char32_type(no_cpp=True)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py
@@ -222,7 +222,7 @@
         F = tp.is_float_type()
         X = tp.is_complex_type()
         I = tp.is_integer_type()
-        assert C == (typename in ('char', 'wchar_t'))
+        assert C == (typename in ('char', 'wchar_t', 'char16_t', 'char32_t'))
         assert F == (typename in ('float', 'double', 'long double'))
         assert X == (typename in ('float _Complex', 'double _Complex'))
         assert I + F + C + X == 1      # one and only one of them is true

From pypy.commits at gmail.com  Mon Jun  5 07:49:37 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 04:49:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: copy over revision
 8426aa942feecfa48d92952654e91788248655b8 (includes several pull request,
 such as """real time profiling""")
Message-ID: <59354551.5888df0a.9e851.1a7a@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91523:cf5d41cbe737
Date: 2017-06-05 07:48 -0400
http://bitbucket.org/pypy/pypy/changeset/cf5d41cbe737/

Log:	copy over revision 8426aa942feecfa48d92952654e91788248655b8
	(includes several pull request, such as """real time profiling""")

diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -214,6 +214,46 @@
     Py_XDECREF(gc_module);
 }
 
+static int emit_all_code_objects_helper(int only_needed, int disable_vmprof)
+{
+    int fd = vmp_profile_fileno();
+
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return -1;
+    }
+
+#if VMPROF_UNIX
+    if ((read(fd, NULL, 0) != 0) && (only_needed != 0)) {
+        PyErr_SetString(PyExc_ValueError,
+                        "file descriptor must be readable to save only needed code objects");
+        return -1;
+    }
+#else
+    if (only_needed) {
+        PyErr_SetString(PyExc_ValueError,
+                        "saving only needed code objects is not supported for windows");
+        return -1;
+    }
+#endif
+
+    if (disable_vmprof) {
+        is_enabled = 0;
+        vmprof_ignore_signals(1);
+    }
+
+#if VMPROF_UNIX
+    if (only_needed)
+        emit_all_code_objects_seen(fd);
+    else
+        emit_all_code_objects();
+#else
+    emit_all_code_objects();
+#endif
+
+    return 0;
+}
+
 static void cpyprof_code_dealloc(PyObject *co)
 {
     if (is_enabled) {
@@ -229,10 +269,11 @@
     int memory = 0;
     int lines = 0;
     int native = 0;
+    int real_time = 0;
     double interval;
     char *p_error;
 
-    if (!PyArg_ParseTuple(args, "id|iii", &fd, &interval, &memory, &lines, &native)) {
+    if (!PyArg_ParseTuple(args, "id|iiii", &fd, &interval, &memory, &lines, &native, &real_time)) {
         return NULL;
     }
 
@@ -251,6 +292,13 @@
         return NULL;
     }
 
+#ifndef VMPROF_UNIX
+    if (real_time) {
+        PyErr_SetString(PyExc_ValueError, "real time profiling is only supported on Linux and MacOS");
+        return NULL;
+    }
+#endif
+
     vmp_profile_lines(lines);
 
     if (!Original_code_dealloc) {
@@ -258,21 +306,20 @@
         PyCode_Type.tp_dealloc = &cpyprof_code_dealloc;
     }
 
-    p_error = vmprof_init(fd, interval, memory, lines, "cpython", native);
+    p_error = vmprof_init(fd, interval, memory, lines, "cpython", native, real_time);
     if (p_error) {
         PyErr_SetString(PyExc_ValueError, p_error);
         return NULL;
     }
 
-    if (vmprof_enable(memory, native) < 0) {
+    if (vmprof_enable(memory, native, real_time) < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
 
     is_enabled = 1;
 
-    Py_INCREF(Py_None);
-    return Py_None;
+    Py_RETURN_NONE;
 }
 
 static PyObject * vmp_is_enabled(PyObject *module, PyObject *noargs) {
@@ -285,66 +332,40 @@
 static PyObject *
 disable_vmprof(PyObject *module, PyObject *args)
 {
-    int fd = vmp_profile_fileno();
     int only_needed = 0;
 
-    if (!PyArg_ParseTuple(args, "|i", &only_needed)) {
+    if (!PyArg_ParseTuple(args, "|i", &only_needed))
         return NULL;
-    }
 
-#if VMPROF_UNIX
-    if ((read(fd, NULL, 0) != 0) && (only_needed != 0)) {
-        PyErr_SetString(PyExc_ValueError,
-                        "file descriptor must be readable to save only needed code objects");
+    if (emit_all_code_objects_helper(only_needed, 1))
         return NULL;
-    }
-#else
-    if (only_needed) {
-        PyErr_SetString(PyExc_ValueError,
-                        "saving only needed code objects is not supported for windows");
-        return NULL;
-    }
-#endif
-
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
-        return NULL;
-    }
-
-    is_enabled = 0;
-    vmprof_ignore_signals(1);
-
-#if VMPROF_UNIX
-    if (only_needed)
-        emit_all_code_objects_seen(fd);
-    else
-        emit_all_code_objects();
-#else
-    emit_all_code_objects();
-#endif
 
     if (vmprof_disable() < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
+
     if (PyErr_Occurred())
         return NULL;
-    Py_INCREF(Py_None);
-    return Py_None;
+
+    Py_RETURN_NONE;
 }
 
 static PyObject *
-write_all_code_objects(PyObject *module, PyObject *noargs)
+write_all_code_objects(PyObject *module, PyObject *args)
 {
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+    int only_needed = 0;
+
+    if (!PyArg_ParseTuple(args, "|i", &only_needed))
         return NULL;
-    }
-    emit_all_code_objects();
+
+    if (emit_all_code_objects_helper(only_needed, 0))
+        return NULL;
+
     if (PyErr_Occurred())
         return NULL;
-    Py_INCREF(Py_None);
-    return Py_None;
+
+    Py_RETURN_NONE;
 }
 
 
@@ -378,7 +399,7 @@
         vmprof_ignore_signals(0);
         return NULL;
     }
-    entry_count = vmp_walk_and_record_stack(tstate->frame, m, MAX_STACK_DEPTH-1, skip, 0);
+    entry_count = vmp_walk_and_record_stack(tstate->frame, m, SINGLE_BUF_SIZE/sizeof(void*)-1, (int)skip, 0);
 
     for (i = 0; i < entry_count; i++) {
         routine_ip = m[i];
@@ -387,16 +408,14 @@
 
     free(m);
 
+    vmprof_ignore_signals(0);
     Py_INCREF(list);
+    return list;
 
+error:
     vmprof_ignore_signals(0);
-    return list;
-error:
     Py_DECREF(list);
-    Py_INCREF(Py_None);
-
-    vmprof_ignore_signals(0);
-    return Py_None;
+    Py_RETURN_NONE;
 }
 
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
@@ -428,13 +447,13 @@
     if (o_srcfile == NULL) goto error;
     //
     return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
+
 error:
     Py_XDECREF(o_name);
     Py_XDECREF(o_lineno);
     Py_XDECREF(o_srcfile);
 
-    Py_INCREF(Py_None);
-    return Py_None;
+    Py_RETURN_NONE;
 }
 #endif
 
@@ -455,18 +474,72 @@
 }
 #endif
 
+
+#ifdef VMPROF_UNIX
+static PyObject *
+insert_real_time_thread(PyObject *module, PyObject * noargs) {
+    ssize_t thread_count;
+
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+
+    if (signal_type != SIGALRM) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not in real time mode");
+        return NULL;
+    }
+
+    while (__sync_lock_test_and_set(&spinlock, 1)) {
+    }
+
+    thread_count = insert_thread(pthread_self(), -1);
+    __sync_lock_release(&spinlock);
+
+    return PyLong_FromSsize_t(thread_count);
+}
+
+static PyObject *
+remove_real_time_thread(PyObject *module, PyObject * noargs) {
+    ssize_t thread_count;
+
+    if (!is_enabled) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
+        return NULL;
+    }
+
+    if (signal_type != SIGALRM) {
+        PyErr_SetString(PyExc_ValueError, "vmprof is not in real time mode");
+        return NULL;
+    }
+
+    while (__sync_lock_test_and_set(&spinlock, 1)) {
+    }
+
+    thread_count = remove_thread(pthread_self(), -1);
+    __sync_lock_release(&spinlock);
+
+    return PyLong_FromSsize_t(thread_count);
+}
+#endif
+
+
 static PyMethodDef VMProfMethods[] = {
     {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
     {"disable", disable_vmprof, METH_VARARGS, "Disable profiling."},
-    {"write_all_code_objects", write_all_code_objects, METH_NOARGS,
+    {"write_all_code_objects", write_all_code_objects, METH_VARARGS,
      "Write eagerly all the IDs of code objects"},
     {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"},
+    {"is_enabled", vmp_is_enabled, METH_NOARGS, "Indicates if vmprof is currently sampling."},
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
     {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
 #endif
-    {"is_enabled", vmp_is_enabled, METH_NOARGS, "Indicates if vmprof is currently sampling."},
 #ifdef VMPROF_UNIX
     {"get_profile_path", vmp_get_profile_path, METH_NOARGS, "Profile path the profiler logs to."},
+    {"insert_real_time_thread", insert_real_time_thread, METH_NOARGS,
+     "Insert a thread into the real time profiling list."},
+    {"remove_real_time_thread", remove_real_time_thread, METH_NOARGS,
+     "Remove a thread from the real time profiling list."},
 #endif
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
diff --git a/rpython/rlib/rvmprof/src/shared/machine.c b/rpython/rlib/rvmprof/src/shared/machine.c
--- a/rpython/rlib/rvmprof/src/shared/machine.c
+++ b/rpython/rlib/rvmprof/src/shared/machine.c
@@ -4,7 +4,6 @@
 #include <stdio.h>
 
 #ifdef VMPROF_UNIX
-#include <string.h>
 #include <unistd.h>
 #include <fcntl.h>
 #endif
diff --git a/rpython/rlib/rvmprof/src/shared/rss_darwin.h b/rpython/rlib/rvmprof/src/shared/rss_darwin.h
--- a/rpython/rlib/rvmprof/src/shared/rss_darwin.h
+++ b/rpython/rlib/rvmprof/src/shared/rss_darwin.h
@@ -24,7 +24,7 @@
 
     kern_return_t error = task_info(mach_task, MACH_TASK_BASIC_INFO, (task_info_t)&taskinfo, &out_count);
     if (error == KERN_SUCCESS) {
-        return taskinfo.resident_size / 1024;
+        return (long)(taskinfo.resident_size / 1024);
     } else {
         return -1;
     }
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -306,7 +306,7 @@
 int _skip_string(int fileno)
 {
     long chars;
-    int count = read(fileno, &chars, sizeof(long));
+    ssize_t count = read(fileno, &chars, sizeof(long));
     //LOG("reading string of %d chars\n", chars);
     if (count <= 0) {
         return 1;
@@ -359,7 +359,7 @@
     return 0;
 }
 
-KHASH_MAP_INIT_INT(ptr, intptr_t)
+KHASH_MAP_INIT_INT64(ptr, int)
 
 void vmp_scan_profile(int fileno, int dump_nat_sym, void *all_code_uids)
 {
@@ -442,7 +442,7 @@
 
                             // if the address has already been dumped,
                             // do not log it again!
-                            it = kh_get(ptr, nat_syms, (intptr_t)addr);
+                            it = kh_get(ptr, nat_syms, (khint64_t)addr);
                             if (it == kh_end(nat_syms)) {
                                 char name[MAXLEN];
                                 char srcfile[MAXLEN];
@@ -453,7 +453,7 @@
                                     LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno);
                                     _dump_native_symbol(fileno, addr, name, lineno, srcfile);
                                     int ret;
-                                    it = kh_put(ptr, nat_syms, (intptr_t)addr, &ret);
+                                    it = kh_put(ptr, nat_syms, (khint64_t)addr, &ret);
                                     kh_value(nat_syms, it) = 1;
                                 }
                             }
diff --git a/rpython/rlib/rvmprof/src/shared/vmp_stack.c b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
--- a/rpython/rlib/rvmprof/src/shared/vmp_stack.c
+++ b/rpython/rlib/rvmprof/src/shared/vmp_stack.c
@@ -15,12 +15,12 @@
 #include "compat.h"
 
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
+
+#ifdef VMPROF_LINUX
 #include "unwind/vmprof_unwind.h"
-
 typedef mcontext_t unw_context_t;
 
 // functions copied from libunwind using dlopen
-
 static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL;
 static int (*unw_step)(unw_cursor_t*) = NULL;
 static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL;
@@ -28,6 +28,9 @@
 static int (*unw_get_proc_name)(unw_cursor_t *, char *, size_t, unw_word_t*) = NULL;
 static int (*unw_is_signal_frame)(unw_cursor_t *) = NULL;
 static int (*unw_getcontext)(unw_context_t *) = NULL;
+#else
+#include <libunwind.h>
+#endif
 
 #endif
 
@@ -44,6 +47,19 @@
 #include <dlfcn.h>
 #endif
 
+int _per_loop(void) {
+    // how many void* are written to the stack trace per loop iterations?
+#ifdef RPYTHON_VMPROF
+    return 2;
+#else
+    if (vmp_profiles_python_lines()) {
+        return 2;
+    }
+    return 1;
+#endif
+}
+
+
 #ifdef PY_TEST
 // for testing only!
 PY_EVAL_RETURN_T * vmprof_eval(PY_STACK_FRAME_T *f, int throwflag) { return NULL; }
@@ -130,19 +146,27 @@
 int vmp_walk_and_record_python_stack_only(PY_STACK_FRAME_T *frame, void ** result,
                                           int max_depth, int depth, intptr_t pc)
 {
-    while (depth < max_depth && frame) {
+    while ((depth + _per_loop()) <= max_depth && frame) {
         frame = _write_python_stack_entry(frame, result, &depth, max_depth);
     }
     return depth;
 }
 
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
-int _write_native_stack(void* addr, void ** result, int depth) {
+int _write_native_stack(void* addr, void ** result, int depth, int max_depth) {
 #ifdef RPYTHON_VMPROF
+    if (depth + 2 >= max_depth) {
+        // bail, do not write to unknown memory
+        return depth;
+    }
     result[depth++] = (void*)VMPROF_NATIVE_TAG;
 #else
     if (vmp_profiles_python_lines()) {
-        // even if we do not log a python stack frame,
+        if (depth + 2 >= max_depth) {
+            // bail, do not write to unknown memory
+            return depth;
+        }
+        // even if we do not log a python line number,
         // we must keep the profile readable
         result[depth++] = 0;
     }
@@ -176,26 +200,36 @@
     // is saved in CPython. _write_python_stack_entry for details.
     //
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    intptr_t func_addr;
+    void * func_addr;
     unw_cursor_t cursor;
     unw_context_t uc;
     unw_proc_info_t pip;
+    int ret;
 
-    if (!vmp_native_enabled()) {
+    if (vmp_native_enabled() == 0) {
         return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
     }
 
-    unw_getcontext(&uc);
-    int ret = unw_init_local(&cursor, &uc);
+    ret = unw_getcontext(&uc);
     if (ret < 0) {
         // could not initialize lib unwind cursor and context
-        return 0;
+        fprintf(stderr, "WARNING: unw_getcontext did not retreive context, switching to python profiling mode \n");
+        vmp_native_disable();
+        return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
+    }
+    ret = unw_init_local(&cursor, &uc);
+    if (ret < 0) {
+        // could not initialize lib unwind cursor and context
+        fprintf(stderr, "WARNING: unw_init_local did not succeed, switching to python profiling mode \n");
+        vmp_native_disable();
+        return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
     }
 
     if (signal < 0) {
         while (signal < 0) {
             int err = unw_step(&cursor);
             if (err <= 0) {
+                fprintf(stderr, "WARNING: did not find signal frame, skipping sample\n");
                 return 0;
             }
             signal++;
@@ -210,6 +244,7 @@
             }
             int err = unw_step(&cursor);
             if (err <= 0) {
+                fprintf(stderr,"WARNING: did not find signal frame, skipping sample\n");
                 return 0;
             }
         }
@@ -227,10 +262,10 @@
 
     int depth = 0;
     PY_STACK_FRAME_T * top_most_frame = frame;
-    while (depth < max_depth) {
+    while ((depth + _per_loop()) <= max_depth) {
         unw_get_proc_info(&cursor, &pip);
 
-        func_addr = pip.start_ip;
+        func_addr = (void*)pip.start_ip;
 
         //{
         //    char name[64];
@@ -269,7 +304,7 @@
             // this is possible because compiler align to 8 bytes.
             //
             if (func_addr != 0x0) {
-                depth = _write_native_stack((void*)(func_addr | 0x1), result, depth);
+                depth = _write_native_stack((void*)(((uint64_t)func_addr) | 0x1), result, depth, max_depth);
             }
         }
 
@@ -277,14 +312,14 @@
         if (err == 0) {
             break;
         } else if (err < 0) {
-            return 0; // this sample is broken, cannot walk it fully
+            // this sample is broken, cannot walk native level... record python level (at least)
+            return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
         }
     }
 
-    return 0; // kill this sample, no python level was found
-#else
+    // if we come here, the found stack trace is removed and only python stacks are recorded
+#endif
     return vmp_walk_and_record_python_stack_only(frame, result, max_depth, 0, pc);
-#endif
 }
 
 int vmp_native_enabled(void) {
@@ -442,7 +477,7 @@
         kr = mach_vm_region(task, &addr, &vmsize, VM_REGION_TOP_INFO,
                           (vm_region_info_t)&topinfo, &count, &obj);
         if (kr == KERN_SUCCESS) {
-            vm_address_t start = addr, end = addr + vmsize;
+            vm_address_t start = (vm_address_t)addr, end = (vm_address_t)(addr + vmsize);
             // dladdr now gives the path of the shared object
             Dl_info info;
             if (dladdr((const void*)start, &info) == 0) {
@@ -484,53 +519,48 @@
 #endif
 #define U_PREFIX "_U"
 #define UL_PREFIX "_UL"
-#else
-#define LIBUNWIND "/usr/lib/system/libunwind.dylib"
-#define PREFIX "unw"
-#define U_PREFIX ""
-#define UL_PREFIX ""
 #endif
 
 int vmp_native_enable(void) {
-    vmp_native_traces_enabled = 1;
-
+#ifdef VMPROF_LINUX
     if (!unw_get_reg) {
-        if (!(libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL))) {
+        if ((libhandle = dlopen(LIBUNWIND, RTLD_LAZY | RTLD_LOCAL)) == NULL) {
             goto bail_out;
         }
-        if (!(unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg"))) {
+        if ((unw_get_reg = dlsym(libhandle, UL_PREFIX PREFIX "_get_reg")) == NULL) {
             goto bail_out;
         }
-        if (!(unw_get_proc_info = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_info"))){
+        if ((unw_get_proc_info = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_info")) == NULL){
             goto bail_out;
         }
-        if (!(unw_get_proc_name = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_name"))){
+        if ((unw_get_proc_name = dlsym(libhandle, UL_PREFIX PREFIX "_get_proc_name")) == NULL){
             goto bail_out;
         }
-        if (!(unw_init_local = dlsym(libhandle, UL_PREFIX PREFIX "_init_local"))) {
+        if ((unw_init_local = dlsym(libhandle, UL_PREFIX PREFIX "_init_local")) == NULL) {
             goto bail_out;
         }
-        if (!(unw_step = dlsym(libhandle, UL_PREFIX PREFIX "_step"))) {
+        if ((unw_step = dlsym(libhandle, UL_PREFIX PREFIX "_step")) == NULL) {
             goto bail_out;
         }
-        if (!(unw_is_signal_frame = dlsym(libhandle, UL_PREFIX PREFIX "_is_signal_frame"))) {
+        if ((unw_is_signal_frame = dlsym(libhandle, UL_PREFIX PREFIX "_is_signal_frame")) == NULL) {
             goto bail_out;
         }
-        if (!(unw_getcontext = dlsym(libhandle, U_PREFIX PREFIX "_getcontext"))) {
+        if ((unw_getcontext = dlsym(libhandle, U_PREFIX PREFIX "_getcontext")) == NULL) {
             goto bail_out;
         }
     }
+#endif
 
-#if defined(__unix__)
-    return vmp_read_vmaps("/proc/self/maps");
-#elif defined(__APPLE__)
-    return vmp_read_vmaps(NULL);
-#endif
+    vmp_native_traces_enabled = 1;
+    return 1;
+
+#ifdef VMPROF_LINUX
 bail_out:
     vmprof_error = dlerror();
     fprintf(stderr, "could not load libunwind at runtime. error: %s\n", vmprof_error);
     vmp_native_traces_enabled = 0;
     return 0;
+#endif
 }
 
 void vmp_native_disable(void) {
@@ -554,7 +584,7 @@
     if (vmp_range_count == 0) {
         return 0;
     }
-    int i = vmp_binary_search_ranges(ip, vmp_ranges, vmp_range_count);
+    int i = vmp_binary_search_ranges(ip, vmp_ranges, (int)vmp_range_count);
     if (i == -1) {
         return 0;
     }
@@ -583,9 +613,9 @@
                 // we found the lower bound
                 i = l - ol;
                 if ((i & 1) == 1) {
-                    return i-1;
+                    return (int)i-1;
                 }
-                return i;
+                return (int)i;
             }
         }
         intptr_t * m = l + i;
@@ -599,7 +629,7 @@
 }
 
 int vmp_ignore_symbol_count(void) {
-    return vmp_range_count;
+    return (int)vmp_range_count;
 }
 
 intptr_t * vmp_ignore_symbols(void) {
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof.h b/rpython/rlib/rvmprof/src/shared/vmprof.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof.h
@@ -26,6 +26,7 @@
 #define PROFILE_LINES  '\x02'
 #define PROFILE_NATIVE '\x04'
 #define PROFILE_RPYTHON '\x08'
+#define PROFILE_REAL_TIME '\x10'
 
 #define DYN_JIT_FLAG 0xbeefbeef
 
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -12,17 +12,82 @@
 #include "vmprof_mt.h"
 #endif
 
+#ifdef VMPROF_LINUX
+#include <syscall.h>
+#endif
+
 #define MAX_FUNC_NAME 1024
 
 static long prepare_interval_usec = 0;
 static long profile_interval_usec = 0;
 
-static int opened_profile(const char *interp_name, int memory, int proflines, int native);
+static int opened_profile(const char *interp_name, int memory, int proflines, int native, int real_time);
 
 #ifdef VMPROF_UNIX
+static int signal_type = SIGPROF;
+static int itimer_type = ITIMER_PROF;
+static pthread_t *threads = NULL;
+static size_t threads_size = 0;
+static size_t thread_count = 0;
+static size_t threads_size_step = 8;
 static struct profbuf_s *volatile current_codes;
 #endif
 
+#ifdef VMPROF_UNIX
+
+static inline ssize_t search_thread(pthread_t tid, ssize_t i) {
+    if (i < 0)
+        i = 0;
+    while (i < thread_count) {
+        if (pthread_equal(threads[i], tid))
+            return i;
+        i++;
+    }
+    return -1;
+}
+
+ssize_t insert_thread(pthread_t tid, ssize_t i) {
+    assert(signal_type == SIGALRM);
+    i = search_thread(tid, i);
+    if (i > 0)
+        return -1;
+    if (thread_count == threads_size) {
+        threads_size += threads_size_step;
+        threads = realloc(threads, sizeof(pid_t) * threads_size);
+        assert(threads != NULL);
+        memset(threads + thread_count, 0, sizeof(pid_t) * threads_size_step);
+    }
+    threads[thread_count++] = tid;
+    return thread_count;
+}
+
+ssize_t remove_thread(pthread_t tid, ssize_t i) {
+    assert(signal_type == SIGALRM);
+    if (thread_count == 0)
+        return -1;
+    if (threads == NULL)
+        return -1;
+    i = search_thread(tid, i);
+    if (i < 0)
+        return -1;
+    threads[i] = threads[--thread_count];
+    threads[thread_count] = 0;
+    return thread_count;
+}
+
+ssize_t remove_threads(void) {
+    assert(signal_type == SIGALRM);
+    if (threads != NULL) {
+        free(threads);
+        threads = NULL;
+    }
+    thread_count = 0;
+    threads_size = 0;
+    return 0;
+}
+
+#endif
+
 #define MAX_STACK_DEPTH   \
     ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *))
 
@@ -64,7 +129,7 @@
 
 RPY_EXTERN
 char *vmprof_init(int fd, double interval, int memory,
-                  int proflines, const char *interp_name, int native)
+                  int proflines, const char *interp_name, int native, int real_time)
 {
     if (!(interval >= 1e-6 && interval < 1.0)) {   /* also if it is NaN */
         return "bad value for 'interval'";
@@ -74,6 +139,13 @@
     if (prepare_concurrent_bufs() < 0)
         return "out of memory";
 #if VMPROF_UNIX
+    if (real_time) {
+        signal_type = SIGALRM;
+        itimer_type = ITIMER_REAL;
+    } else {
+        signal_type = SIGPROF;
+        itimer_type = ITIMER_PROF;
+    }
     current_codes = NULL;
     assert(fd >= 0);
 #else
@@ -85,14 +157,14 @@
     }
 #endif
     vmp_set_profile_fileno(fd);
-    if (opened_profile(interp_name, memory, proflines, native) < 0) {
+    if (opened_profile(interp_name, memory, proflines, native, real_time) < 0) {
         vmp_set_profile_fileno(0);
         return strerror(errno);
     }
     return NULL;
 }
 
-static int opened_profile(const char *interp_name, int memory, int proflines, int native)
+static int opened_profile(const char *interp_name, int memory, int proflines, int native, int real_time)
 {
     int success;
     int bits;
@@ -119,7 +191,7 @@
     header.interp_name[1] = '\x00';
     header.interp_name[2] = VERSION_TIMESTAMP;
     header.interp_name[3] = memory*PROFILE_MEMORY + proflines*PROFILE_LINES + \
-                            native*PROFILE_NATIVE;
+                            native*PROFILE_NATIVE + real_time*PROFILE_REAL_TIME;
 #ifdef RPYTHON_VMPROF
     header.interp_name[3] += PROFILE_RPYTHON;
 #endif
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -48,11 +48,14 @@
 #include "rss_darwin.h"
 #endif
 
+#if VMPROF_LINUX
+#include <syscall.h>
+#endif
 
 /************************************************************/
 
 static void *(*mainloop_get_virtual_ip)(char *) = 0;
-static int opened_profile(const char *interp_name, int memory, int proflines, int native);
+static int opened_profile(const char *interp_name, int memory, int proflines, int native, int real_time);
 static void flush_codes(void);
 
 /************************************************************/
@@ -94,14 +97,19 @@
 {
     PY_STACK_FRAME_T * frame;
 #ifdef RPYTHON_VMPROF
-    // do nothing here, 
+    // do nothing here,
     frame = (PY_STACK_FRAME_T*)current;
 #else
-    if (!current) {
+    if (current == NULL) {
+        fprintf(stderr, "WARNING: get_stack_trace, current is NULL\n");
         return 0;
     }
     frame = current->frame;
 #endif
+    if (frame == NULL) {
+        fprintf(stderr, "WARNING: get_stack_trace, frame is NULL\n");
+        return 0;
+    }
     return vmp_walk_and_record_stack(frame, result, max_depth, 1, pc);
 }
 
@@ -155,11 +163,12 @@
     PyThreadState * state;
     long mythread_id;
 
+    mythread_id = PyThread_get_thread_ident();
     istate = PyInterpreterState_Head();
     if (istate == NULL) {
+        fprintf(stderr, "WARNING: interp state head is null (for thread id %d)\n", mythread_id);
         return NULL;
     }
-    mythread_id = PyThread_get_thread_ident();
     // fish fish fish, it will NOT lock the keymutex in pythread
     do {
         state = PyInterpreterState_ThreadHead(istate);
@@ -171,16 +180,61 @@
     } while ((istate = PyInterpreterState_Next(istate)) != NULL);
 
     // uh? not found?
+    fprintf(stderr, "WARNING: cannot find thread state (for thread id %d), sample will be thrown away\n", mythread_id);
     return NULL;
 }
 #endif
 
+#ifdef VMPROF_UNIX
+static int broadcast_signal_for_threads(void)
+{
+    int done = 1;
+    size_t i = 0;
+    pthread_t self = pthread_self();
+    pthread_t tid;
+    while (i < thread_count) {
+        tid = threads[i];
+        if (pthread_equal(tid, self)) {
+            done = 0;
+        } else if (pthread_kill(tid, SIGALRM)) {
+            remove_thread(tid, i);
+        }
+        i++;
+    }
+    return done;
+}
+#endif
+
+#ifdef VMPROF_LINUX
+static inline int is_main_thread(void)
+{
+    pid_t pid = getpid();
+    pid_t tid = (pid_t) syscall(SYS_gettid);
+    return (pid == tid);
+}
+#endif
+
+#ifdef VMPROF_APPLE
+static inline int is_main_thread(void)
+{
+    return pthread_main_np();
+}
+#endif
+
 static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext)
 {
     int commit;
     PY_THREAD_STATE_T * tstate = NULL;
     void (*prevhandler)(int);
+
 #ifndef RPYTHON_VMPROF
+
+    // Even though the docs say that this function call is for 'esoteric use'
+    // it seems to be correctly set when the interpreter is teared down!
+    if (!Py_IsInitialized()) {
+        return;
+    }
+
     // TERRIBLE HACK AHEAD
     // on OS X, the thread local storage is sometimes uninitialized
     // when the signal handler runs - it means it's impossible to read errno
@@ -193,6 +247,21 @@
     // get_current_thread_state returns a sane result
     while (__sync_lock_test_and_set(&spinlock, 1)) {
     }
+
+#ifdef VMPROF_UNIX
+    // SIGNAL ABUSE AHEAD
+    // On linux, the prof timer will deliver the signal to the thread which triggered the timer,
+    // because these timers are based on process and system time, and as such, are thread-aware.
+    // For the real timer, the signal gets delivered to the main thread, seemingly always.
+    // Consequently if we want to sample multiple threads, we need to forward this signal.
+    if (signal_type == SIGALRM) {
+        if (is_main_thread() && broadcast_signal_for_threads()) {
+            __sync_lock_release(&spinlock);
+            return;
+        }
+    }
+#endif
+
     prevhandler = signal(SIGSEGV, &segfault_handler);
     int fault_code = setjmp(restore_point);
     if (fault_code == 0) {
@@ -226,6 +295,7 @@
             if (commit) {
                 commit_buffer(fd, p);
             } else {
+                fprintf(stderr, "WARNING: canceled buffer, no stack trace was written %d\n", is_enabled);
                 cancel_buffer(p);
             }
         }
@@ -250,7 +320,7 @@
     sa.sa_sigaction = sigprof_handler;
     sa.sa_flags = SA_RESTART | SA_SIGINFO;
     if (sigemptyset(&sa.sa_mask) == -1 ||
-        sigaction(SIGPROF, &sa, NULL) == -1)
+        sigaction(signal_type, &sa, NULL) == -1)
         return -1;
     return 0;
 }
@@ -262,7 +332,7 @@
     ign_sigint.sa_flags = 0;
     sigemptyset(&ign_sigint.sa_mask);
 
-    if (sigaction(SIGPROF, &ign_sigint, NULL) < 0) {
+    if (sigaction(signal_type, &ign_sigint, NULL) < 0) {
         fprintf(stderr, "Could not remove the signal handler (for profiling)\n");
         return -1;
     }
@@ -273,9 +343,9 @@
 {
     static struct itimerval timer;
     timer.it_interval.tv_sec = 0;
-    timer.it_interval.tv_usec = profile_interval_usec;
+    timer.it_interval.tv_usec = (int)profile_interval_usec;
     timer.it_value = timer.it_interval;
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0)
+    if (setitimer(itimer_type, &timer, NULL) != 0)
         return -1;
     return 0;
 }
@@ -284,7 +354,7 @@
     static struct itimerval timer;
     timerclear(&(timer.it_interval));
     timerclear(&(timer.it_value));
-    if (setitimer(ITIMER_PROF, &timer, NULL) != 0) {
+    if (setitimer(itimer_type, &timer, NULL) != 0) {
         fprintf(stderr, "Could not disable the signal handler (for profiling)\n");
         return -1;
     }
@@ -354,7 +424,7 @@
 #endif
 
 RPY_EXTERN
-int vmprof_enable(int memory, int native)
+int vmprof_enable(int memory, int native, int real_time)
 {
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
     init_cpyprof(native);
@@ -364,6 +434,10 @@
     profile_interval_usec = prepare_interval_usec;
     if (memory && setup_rss() == -1)
         goto error;
+#if VMPROF_UNIX
+    if (real_time && insert_thread(pthread_self(), -1) == -1)
+        goto error;
+#endif
     if (install_pthread_atfork_hooks() == -1)
         goto error;
     if (install_sigprof_handler() == -1)
@@ -409,6 +483,10 @@
         return -1;
     if (remove_sigprof_handler() == -1)
         return -1;
+#ifdef VMPROF_UNIX
+    if ((signal_type == SIGALRM) && remove_threads() == -1)
+        return -1;
+#endif
     flush_codes();
     if (shutdown_concurrent_bufs(vmp_profile_fileno()) < 0)
         return -1;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main_win32.h
@@ -160,7 +160,7 @@
 }
 
 RPY_EXTERN
-int vmprof_enable(int memory, int native)
+int vmprof_enable(int memory, int native, int real_time)
 {
     if (!thread_started) {
         if (!CreateThread(NULL, 0, vmprof_mainloop, NULL, 0, NULL)) {

From pypy.commits at gmail.com  Mon Jun  5 08:05:41 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 05:05:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: adjust _vmprof.enable parameters to
 carry real_time over to the vmprof C library
Message-ID: <59354915.c18b1c0a.fae5e.2b06@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91524:5a98d3aa0153
Date: 2017-06-05 08:04 -0400
http://bitbucket.org/pypy/pypy/changeset/5a98d3aa0153/

Log:	adjust _vmprof.enable parameters to carry real_time over to the
	vmprof C library

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -51,8 +51,8 @@
     return OperationError(w_VMProfError, space.newtext(e.msg))
 
 
- at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int)
-def enable(space, fileno, period, memory, lines, native):
+ at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int, real_time=int)
+def enable(space, fileno, period, memory, lines, native, real_time):
     """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
     opened for writing.  *The file descriptor must remain open at least
     until disable() is called.*
@@ -66,7 +66,7 @@
     #                             "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
-        rvmprof.enable(fileno, period, memory, native)
+        rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:
         raise VMProfError(space, e)
 
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -86,7 +86,7 @@
                                   [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
                                    rffi.CCHARP, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
-    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT],
+    vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT, rffi.INT],
                                     rffi.INT,
                                     compilation_info=eci,
                                     save_err=rffi.RFFI_SAVE_ERRNO)
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -29,7 +29,7 @@
 RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
                      int lines, const char *interp_name, int native);
 RPY_EXTERN void vmprof_ignore_signals(int);
-RPY_EXTERN int vmprof_enable(int memory, int native);
+RPY_EXTERN int vmprof_enable(int memory, int native, int real_time);
 RPY_EXTERN int vmprof_disable(void);
 RPY_EXTERN int vmprof_register_virtual_function(char *, intptr_t, int);
 RPY_EXTERN void* vmprof_stack_new(void);

From pypy.commits at gmail.com  Mon Jun  5 09:07:26 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 06:07:26 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix tests and scatter real_time
 parameter to other functions needed
Message-ID: <5935578e.d18bdf0a.52aba.7413@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91525:82f30247c9bb
Date: 2017-06-05 09:06 -0400
http://bitbucket.org/pypy/pypy/changeset/82f30247c9bb/

Log:	fix tests and scatter real_time parameter to other functions needed

diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py
--- a/pypy/module/_vmprof/test/test__vmprof.py
+++ b/pypy/module/_vmprof/test/test__vmprof.py
@@ -60,7 +60,7 @@
         import _vmprof
         gc.collect()  # try to make the weakref list deterministic
         gc.collect()  # by freeing all dead code objects
-        _vmprof.enable(tmpfileno, 0.01, 0, 0, 0)
+        _vmprof.enable(tmpfileno, 0.01, 0, 0, 0, 0)
         _vmprof.disable()
         s = open(self.tmpfilename, 'rb').read()
         no_of_codes = count(s)
@@ -73,7 +73,7 @@
 
         gc.collect()
         gc.collect()
-        _vmprof.enable(tmpfileno2, 0.01, 0, 0, 0)
+        _vmprof.enable(tmpfileno2, 0.01, 0, 0, 0, 0)
 
         exec """def foo2():
             pass
@@ -88,18 +88,18 @@
 
     def test_enable_ovf(self):
         import _vmprof
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0, 0, 0, 0)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5, 0, 0, 0)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300, 0, 0, 0)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 0, 0, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, -2.5, 0, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300, 0, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, 1e300 * 1e300, 0, 0, 0, 0)
         NaN = (1e300*1e300) / (1e300*1e300)
-        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0)
+        raises(_vmprof.VMProfError, _vmprof.enable, 2, NaN, 0, 0, 0, 0)
 
     def test_is_enabled(self):
         import _vmprof
         tmpfile = open(self.tmpfilename, 'wb')
         assert _vmprof.is_enabled() is False
-        _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0)
+        _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0, 0)
         assert _vmprof.is_enabled() is True
         _vmprof.disable()
         assert _vmprof.is_enabled() is False
@@ -108,7 +108,7 @@
         import _vmprof
         tmpfile = open(self.tmpfilename, 'wb')
         assert _vmprof.get_profile_path() is None
-        _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0)
+        _vmprof.enable(tmpfile.fileno(), 0.01, 0, 0, 0, 0)
         path = _vmprof.get_profile_path()
         if path != tmpfile.name:
             with open(path, "rb") as fd1:
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -33,8 +33,8 @@
         return code._vmprof_unique_id
     return 0
 
-def enable(fileno, interval, memory=0, native=0):
-    _get_vmprof().enable(fileno, interval, memory, native)
+def enable(fileno, interval, memory=0, native=0, real_time=0):
+    _get_vmprof().enable(fileno, interval, memory, native, real_time)
 
 def disable():
     _get_vmprof().disable()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -84,7 +84,7 @@
     eci = global_eci
     vmprof_init = rffi.llexternal("vmprof_init",
                                   [rffi.INT, rffi.DOUBLE, rffi.INT, rffi.INT,
-                                   rffi.CCHARP, rffi.INT],
+                                   rffi.CCHARP, rffi.INT, rffi.INT],
                                   rffi.CCHARP, compilation_info=eci)
     vmprof_enable = rffi.llexternal("vmprof_enable", [rffi.INT, rffi.INT, rffi.INT],
                                     rffi.INT,
diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py
--- a/rpython/rlib/rvmprof/rvmprof.py
+++ b/rpython/rlib/rvmprof/rvmprof.py
@@ -124,7 +124,7 @@
         self._gather_all_code_objs = gather_all_code_objs
 
     @jit.dont_look_inside
-    def enable(self, fileno, interval, memory=0, native=0):
+    def enable(self, fileno, interval, memory=0, native=0, real_time=0):
         """Enable vmprof.  Writes go to the given 'fileno'.
         The sampling interval is given by 'interval' as a number of
         seconds, as a float which must be smaller than 1.0.
@@ -138,12 +138,12 @@
             native = 0 # force disabled on Windows
         lines = 0 # not supported on PyPy currently
 
-        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native)
+        p_error = self.cintf.vmprof_init(fileno, interval, lines, memory, "pypy", native, real_time)
         if p_error:
             raise VMProfError(rffi.charp2str(p_error))
 
         self._gather_all_code_objs()
-        res = self.cintf.vmprof_enable(memory, native)
+        res = self.cintf.vmprof_enable(memory, native, real_time)
         if res < 0:
             raise VMProfError(os.strerror(rposix.get_saved_errno()))
         self.is_enabled = True
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -27,7 +27,7 @@
 #endif
 
 RPY_EXTERN char *vmprof_init(int fd, double interval, int memory,
-                     int lines, const char *interp_name, int native);
+                     int lines, const char *interp_name, int native, int real_time);
 RPY_EXTERN void vmprof_ignore_signals(int);
 RPY_EXTERN int vmprof_enable(int memory, int native, int real_time);
 RPY_EXTERN int vmprof_disable(void);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -6,6 +6,7 @@
 
 #include <stddef.h>
 #include <time.h>
+#include <stdlib.h>
 
 #ifndef VMPROF_WINDOWS
 #include <sys/time.h>
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -295,7 +295,11 @@
             if (commit) {
                 commit_buffer(fd, p);
             } else {
+#ifndef RPYTHON_VMPROF
                 fprintf(stderr, "WARNING: canceled buffer, no stack trace was written %d\n", is_enabled);
+#else
+                fprintf(stderr, "WARNING: canceled buffer, no stack trace was written\n");
+#endif
                 cancel_buffer(p);
             }
         }

From pypy.commits at gmail.com  Mon Jun  5 10:00:58 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 07:00:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: introduce read_exactly as proposed by
 njs, handles EINTR
Message-ID: <5935641a.43861c0a.2c275.b272@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91526:9a792fd023fd
Date: 2017-06-05 10:00 -0400
http://bitbucket.org/pypy/pypy/changeset/9a792fd023fd/

Log:	introduce read_exactly as proposed by njs, handles EINTR

diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -207,7 +207,9 @@
 }
 #endif
 
+static
 struct backtrace_state * bstate = NULL;
+
 int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno, char * srcfile, int srcfile_len) {
 #ifdef __APPLE__
     Dl_info dlinfo;
@@ -303,10 +305,34 @@
     lseek(fileno, pos_before, SEEK_SET);
 }
 
+static
+int cannot_read_profile = 0;
+
+ssize_t read_exactly(int fileno, void * buf, ssize_t size) {
+    assert(size >= 0 && "size parameter must be positive");
+
+    ssize_t r = 0;
+    if ((r = read(fileno, buf, (size_t)size)) == size) {
+        return r;
+    }
+
+    if (r == -1) {
+        if (errno == EINTR) {
+            if ((r = read(fileno, buf, (size_t)size)) == size) {
+                return r;
+            }
+        }
+    }
+
+    fprintf(stderr, "unhandled error in read_exactly. cannot proceed! could not read %d bytes", size);
+    cannot_read_profile = 1;
+    return -1;
+}
+
 int _skip_string(int fileno)
 {
     long chars;
-    ssize_t count = read(fileno, &chars, sizeof(long));
+    ssize_t count = read_exactly(fileno, &chars, sizeof(long));
     //LOG("reading string of %d chars\n", chars);
     if (count <= 0) {
         return 1;
@@ -319,7 +345,7 @@
 int _skip_header(int fileno, int * version, int * flags)
 {
     unsigned char r[4];
-    (void)read(fileno, r, 4);
+    (void)read_exactly(fileno, r, 4);
     unsigned char count = r[3];
     *version = (r[0] & 0xff) << 8 | (r[1] & 0xff);
     *flags = r[2];
@@ -330,14 +356,14 @@
 long _read_word(int fileno)
 {
     long w;
-    (void)read(fileno, &w, WORD_SIZE);
+    (void)read_exactly(fileno, &w, WORD_SIZE);
     return w;
 }
 
 void * _read_addr(int fileno)
 {
     void * a;
-    (void)read(fileno, &a, ADDR_SIZE);
+    (void)read_exactly(fileno, &a, ADDR_SIZE);
     return a;
 }
 
@@ -378,7 +404,11 @@
     lseek(fileno, 5*WORD_SIZE, SEEK_SET);
 
     while (1) {
-        count = read(fileno, &marker, 1);
+        if (cannot_read_profile == 1) {
+            cannot_read_profile = 0;
+            break;
+        }
+        count = read_exactly(fileno, &marker, 1);
         if (count <= 0) {
             break;
         }

From pypy.commits at gmail.com  Mon Jun  5 10:14:17 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 05 Jun 2017 07:14:17 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix tests
Message-ID: <59356739.d1d71c0a.1a6d1.abde@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91527:11a97e2aa4be
Date: 2017-06-05 15:13 +0100
http://bitbucket.org/pypy/pypy/changeset/11a97e2aa4be/

Log:	fix tests

diff --git a/pypy/module/struct/test/test_struct.py b/pypy/module/struct/test/test_struct.py
--- a/pypy/module/struct/test/test_struct.py
+++ b/pypy/module/struct/test/test_struct.py
@@ -394,9 +394,9 @@
         assert self.struct.unpack_from("ii", memoryview(b)[2:]) == (17, 42)
         assert self.struct.unpack_from("ii", memoryview(b), 2) == (17, 42)
         exc = raises(TypeError, self.struct.unpack_from, "ii", 123)
-        assert str(exc.value) == "'int' does not support the buffer interface"
+        assert str(exc.value) == "a bytes-like object is required, not int"
         exc = raises(TypeError, self.struct.unpack_from, "ii", None)
-        assert str(exc.value) == "'NoneType' does not support the buffer interface"
+        assert str(exc.value) == "a bytes-like object is required, not None"
         exc = raises(self.struct.error, self.struct.unpack_from, "ii", b'')
         assert str(exc.value) == "unpack_from requires a buffer of at least 8 bytes"
         exc = raises(self.struct.error, self.struct.unpack_from, "ii", memoryview(b''))
@@ -627,7 +627,7 @@
     def test_unpack_array(self):
         import array
         data = self.struct.pack("iii", 0, 42, 43)
-        buf = array.array('c', data)
+        buf = array.array('B', data)
         assert self.struct.unpack("iii", buf) == (0, 42, 43)
 
     def test_pack_into_bytearray(self):

From pypy.commits at gmail.com  Mon Jun  5 11:00:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 05 Jun 2017 08:00:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: 2to3fy tests
Message-ID: <5935721b.eb8edf0a.460a.add4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91528:2ce7423bee79
Date: 2017-06-05 15:53 +0100
http://bitbucket.org/pypy/pypy/changeset/2ce7423bee79/

Log:	2to3fy tests

diff --git a/pypy/module/pypyjit/test_pypy_c/test_buffers.py b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
--- a/pypy/module/pypyjit/test_pypy_c/test_buffers.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_buffers.py
@@ -6,8 +6,8 @@
         def main(n):
             import re
             import array
-            p = re.compile('.+')
-            a = array.array('c', 'test' * 1000)
+            p = re.compile(b'.+')
+            a = array.array('B', b'test' * 1000)
             i = 0
             while i < n:
                 i += 1
@@ -30,7 +30,7 @@
         def main(n):
             import _struct as struct
             import array
-            a = array.array('c', struct.pack('i', 42))
+            a = array.array('B', struct.pack('i', 42))
             i = 0
             while i < n:
                 i += 1
diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -154,7 +154,7 @@
         def main(n):
             import array
             import struct
-            buf = array.array('b', '\x00'*8)
+            buf = array.array('b', b'\x00'*8)
             i = 1
             while i < n:
                 struct.pack_into("h", buf, 4, i)     # ID: pack_into

From pypy.commits at gmail.com  Mon Jun  5 14:13:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 11:13:35 -0700 (PDT)
Subject: [pypy-commit] pypy default: move lto to build option,
 default is False since some gcc versions produce bogus code
Message-ID: <59359f4f.f4a0df0a.fc59b.43e3@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91529:c6a3a65d4975
Date: 2017-06-05 21:12 +0300
http://bitbucket.org/pypy/pypy/changeset/c6a3a65d4975/

Log:	move lto to build option, default is False since some gcc versions
	produce bogus code

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -202,6 +202,9 @@
     BoolOption("lldebug0",
                "If true, makes an lldebug0 build", default=False,
                cmdline="--lldebug0"),
+    BoolOption("lto", "enable link time optimization",
+               default=False, cmdline="--lto",
+               requires=[("translation.gcrootfinder", "shadowstack")]),
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -134,7 +134,7 @@
             cflags = tuple(self.cflags) + tuple(self.standalone_only)
 
         # xxx check which compilers accept this option or not
-        if not config or config.translation.gcrootfinder != 'asmgcc':
+        if config and config.translation.lto:
             cflags = ('-flto',) + cflags
 
         m = GnuMakefile(path)

From pypy.commits at gmail.com  Mon Jun  5 14:19:33 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 11:19:33 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix for translation,
 I should have included errno.h. translation started annotation
Message-ID: <5935a0b5.493e1c0a.f22ce.d26d@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91530:beebaffe3671
Date: 2017-06-05 14:18 -0400
http://bitbucket.org/pypy/pypy/changeset/beebaffe3671/

Log:	fix for translation, I should have included errno.h. translation
	started annotation

diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -11,6 +11,7 @@
 
 #include <assert.h>
 #include <dlfcn.h>
+#include <errno.h>
 
 #if defined(VMPROF_LINUX)
 #include <link.h>
@@ -317,7 +318,7 @@
     }
 
     if (r == -1) {
-        if (errno == EINTR) {
+        while (errno == EINTR) {
             if ((r = read(fileno, buf, (size_t)size)) == size) {
                 return r;
             }

From pypy.commits at gmail.com  Mon Jun  5 15:53:32 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 12:53:32 -0700 (PDT)
Subject: [pypy-commit] pypy default: clean up many sphinx warnings,
 update copyright and version
Message-ID: <5935b6bc.0d451c0a.eff10.dedf@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91531:66ce5daf5ff9
Date: 2017-06-05 22:52 +0300
http://bitbucket.org/pypy/pypy/changeset/66ce5daf5ff9/

Log:	clean up many sphinx warnings, update copyright and version

diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.4'
+version = '5.8'
 # The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -60,7 +60,7 @@
 The interface for full finalizers is made with PyPy in mind, but should
 be generally useful.
 
-The idea is that you subclass the ``rgc.FinalizerQueue`` class::
+The idea is that you subclass the ``rgc.FinalizerQueue`` class:
 
 * You must give a class-level attribute ``base_class``, which is the
   base class of all instances with a finalizer.  (If you need
diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst
--- a/pypy/doc/discussion/rawrefcount.rst
+++ b/pypy/doc/discussion/rawrefcount.rst
@@ -68,10 +68,12 @@
 and O = list of links created with rawrefcount.create_link_pyobj().
 The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all
 the data is in the PyObjects, and all outsite references (if any) are
-in C, as "PyObject *" fields.
+in C, as ``PyObject *`` fields.
 
 So, during the collection we do this about P links:
 
+.. code-block:: python
+
     for (p, ob) in P:
         if ob->ob_refcnt != REFCNT_FROM_PYPY
                and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT:
@@ -80,6 +82,8 @@
 At the end of the collection, the P and O links are both handled like
 this:
 
+.. code-block:: python
+
     for (p, ob) in P + O:
         if p is not surviving:    # even if 'ob' might be surviving
             unlink p and ob
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -30,12 +30,22 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+CPython 3.5 compatible versions
+-------------------------------
+
+.. toctree::
+
+   whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.8.0.rst
+   whatsnew-pypy3-5.7.0.rst
+
 CPython 3.3 compatible versions
 -------------------------------
 
 .. toctree::
 
    whatsnew-pypy3-5.5.0.rst
+   whatsnew-pypy3-5.1.1-alpha1.rst
 
 CPython 3.2 compatible versions
 -------------------------------
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -12,6 +12,7 @@
 and using pip. 
 
 .. _prebuilt-pypy:
+
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -250,12 +250,12 @@
 .. py:function:: newunicode(ustr)
 
    Creates a Unicode string from an rpython unicode string.
-   This method may disappear soon and be replaced by :py:function:`newutf8()`.
+   This method may disappear soon and be replaced by :py:function::`newutf8`.
 
 .. py:function:: newutf8(bytestr)
 
    Creates a Unicode string from an rpython byte string, decoded as
-   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function::`newtext`.
 
 Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
 `pypy/objspace/std/objspace.py`.
@@ -302,9 +302,9 @@
 
 .. py:function:: unicode_w(w_x)
 
-   Takes an application level :py:class:`unicode` and return an
+   Takes an application level :py:class::`unicode` and return an
    interpreter-level unicode string.  This method may disappear soon and
-   be replaced by :py:function:`text_w()`.
+   be replaced by :py:function::`text_w`.
 
 .. py:function:: float_w(w_x)
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -171,7 +171,7 @@
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
-   `malloc_nursery`, to move values from registers into other registers
+    `malloc_nursery`, to move values from registers into other registers
     instead of to the stack.
 
   * More tightly pack the stack when calling with `release gil`
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -140,7 +140,7 @@
     preamble
   * In JIT residual calls, if the called function starts with a fast-path like
     ``if x.foo != 0: return x.foo``, then inline the check before doing the
-     ``CALL``.
+    ``CALL``.
   * Ensure ``make_inputargs`` fails properly when given arguments with type 
     information
   * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,8 +10,8 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-This release enables `profile guided optimization` of the base interpreter,
-which may make unjitted code run faster.
+This release enables link-time optimization and `profile guided optimization` 
+of the base interpreter, which may make unjitted code run faster.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -6,6 +6,7 @@
 .. startrev: 91904d5c5188
 
 .. branch: use_min_scalar
+
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
 
 .. branch: stdlib-2.7.10
@@ -15,6 +16,7 @@
 .. branch: issue2062
 
 .. branch: disable-unroll-for-short-loops
+
 The JIT no longer performs loop unrolling if the loop compiles to too much code.
 
 .. branch: run-create_cffi_imports
@@ -32,9 +34,11 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+
 Part of cffi 1.2.
 
 .. branch: cffi-new-allocator
+
 Part of cffi 1.2.
 
 .. branch: unicode-dtype
diff --git a/pypy/doc/whatsnew-4.0.0.rst b/pypy/doc/whatsnew-4.0.0.rst
--- a/pypy/doc/whatsnew-4.0.0.rst
+++ b/pypy/doc/whatsnew-4.0.0.rst
@@ -6,23 +6,28 @@
 .. startrev: 3a8f5481dab4
 
 .. branch: keys_with_hash
+
 Improve the performance of ``dict.update()`` and a bunch of methods from
 sets, by reusing the hash value stored in one dict when inspecting
 or changing another dict with that key.
 
 .. branch: optresult-unroll 
+
 A major refactoring of the ``ResOperations`` that kills Box. Also rewrote
 unrolling to enable future enhancements.  Should improve warmup time
 by 20% or so.
 
 .. branch: optimize-cond-call
+
 Optimize common sequences of operations like
 ``int_lt/cond_call`` in the JIT backends
 
 .. branch: missing_openssl_include
+
 Fix for missing headers in OpenBSD, already applied in downstream ports
 
 .. branch: gc-more-incremental
+
 Remove a source of non-incremental-ness in the GC: now
 ``external_malloc()`` no longer runs ``gc_step_until()`` any more. If there
 is a currently-running major collection, we do only so many steps
@@ -32,11 +37,13 @@
 keep adding up between them.
 
 .. branch: remember-tracing-counts
+
 Reenable jithooks
 
 .. branch: detect_egd2
 
 .. branch: shadowstack-no-move-2
+
 Issue #2141: fix a crash on Windows and OS/X and ARM when running
 at least 20 threads.
 
@@ -55,6 +62,7 @@
 floats, cf. issue #2148.
 
 .. branch: cffi-stdcall
+
 Win32: support ``__stdcall`` in CFFI.
 
 .. branch: callfamily
@@ -93,6 +101,7 @@
 .. branch: osx-libffi
 
 .. branch: lazy-fast2locals
-improve the performance of simple trace functions by lazily calling
+
+Improve the performance of simple trace functions by lazily calling
 ``fast2locals`` and ``locals2fast`` only if ``f_locals`` is actually accessed.
 
diff --git a/pypy/doc/whatsnew-5.0.0.rst b/pypy/doc/whatsnew-5.0.0.rst
--- a/pypy/doc/whatsnew-5.0.0.rst
+++ b/pypy/doc/whatsnew-5.0.0.rst
@@ -192,6 +192,7 @@
 Fix boolean-array indexing in micronumpy
 
 .. branch: numpy_partition
+
 Support ndarray.partition() as an app-level function numpy.core._partition_use,
 provided as a cffi wrapper to upstream's implementation in the pypy/numpy repo
 
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -29,6 +29,7 @@
 upstream numpy via cpyext, so we created (yet another) fork of numpy at 
 github.com/pypy/numpy with the needed changes. Among the significant changes 
 to cpyext:
+
   - allow c-snippet tests to be run with -A so we can verify we are compatible
   - fix many edge cases exposed by fixing tests to run with -A
   - issequence() logic matches cpython
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -6,10 +6,12 @@
 .. startrev: 873218a739f1
 
 .. 418b05f95db5
+
 Improve CPython compatibility for ``is``. Now code like ``if x is ():``
 works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
 .. pull request #455
+
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 
 .. branch: fix-gen-dfa
@@ -36,9 +38,11 @@
 compatible.
 
 .. branch: pyfile-tell
+
 Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
 
 .. branch: rw-PyString_AS_STRING
+
 Allow rw access to the char* returned from PyString_AS_STRING, also refactor
 PyStringObject to look like cpython's and allow subclassing PyString_Type and
 PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.6.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -6,18 +6,22 @@
 .. startrev: 522736f816dc
 
 .. branch: rpython-resync
+
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
 .. branch: buffer-interface
+
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
 .. branch: force-virtual-state
+
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
 
 .. branch: conditional_call_value_3
+
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
@@ -58,6 +62,7 @@
 
 
 .. fb6bb835369e
+
 Change the ``timeit`` module: it now prints the average time and the standard
 deviation over 7 runs by default, instead of the minimum. The minimum is often
 misleading.
@@ -69,9 +74,6 @@
 
 .. branch: Tiberiumk/fix-2412-1476011166874
 .. branch: redirect-assembler-jitlog
-
-
-
 .. branch: stdlib-2.7.12
 
 Update stdlib to version 2.7.12
diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
--- a/pypy/doc/whatsnew-pypy3-5.8.0.rst
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -6,6 +6,7 @@
 .. startrev: afbf09453369
 
 .. branch: mtest
+
 Use "<python> -m test" to run the CPython test suite, as documented by CPython,
 instead of our outdated regrverbose.py script.
 

From pypy.commits at gmail.com  Mon Jun  5 15:58:06 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 12:58:06 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: move lto to build option,
 default is False since some gcc versions produce bogus code
Message-ID: <5935b7ce.d1d71c0a.1a6d1.0d0c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91533:df2a7a142dd4
Date: 2017-06-05 22:55 +0300
http://bitbucket.org/pypy/pypy/changeset/df2a7a142dd4/

Log:	move lto to build option, default is False since some gcc versions
	produce bogus code (grafted from
	c6a3a65d4975769d0767d2d682e4d9741a8042c4)

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -202,6 +202,9 @@
     BoolOption("lldebug0",
                "If true, makes an lldebug0 build", default=False,
                cmdline="--lldebug0"),
+    BoolOption("lto", "enable link time optimization",
+               default=False, cmdline="--lto",
+               requires=[("translation.gcrootfinder", "shadowstack")]),
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -134,7 +134,7 @@
             cflags = tuple(self.cflags) + tuple(self.standalone_only)
 
         # xxx check which compilers accept this option or not
-        if not config or config.translation.gcrootfinder != 'asmgcc':
+        if config and config.translation.lto:
             cflags = ('-flto',) + cflags
 
         m = GnuMakefile(path)

From pypy.commits at gmail.com  Mon Jun  5 15:58:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:08 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: Fix: 'tp_descr_get(self,
 NULL, type)' used to give a real undefined
Message-ID: <5935b7d0.86871c0a.fa7c8.0d21@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r91534:59850276ad66
Date: 2017-06-05 22:56 +0300
http://bitbucket.org/pypy/pypy/changeset/59850276ad66/

Log:	Fix: 'tp_descr_get(self, NULL, type)' used to give a real undefined
	value for the second argument if implemented in Python (grafted from
	07257b27db4b6e747d50f24108835f3ed3f62a79)

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -51,7 +51,7 @@
         w_descr = space.appexec([], """():
             class Descr(object):
                 def __get__(self, obj, type):
-                    return 42
+                    return 42 + (obj is None)
                 def __set__(self, obj, value):
                     obj.append('set')
                 def __delete__(self, obj):
@@ -73,6 +73,11 @@
             space, py_descrtype.c_tp_descr_set,
             py_descr, py_obj, None) == 0
         assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+        #
+        # unbound __get__(self, NULL, type)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, None, space.w_int)
+        assert space.int_w(w_res) == 43
 
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -111,6 +111,8 @@
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    if w_obj is None:
+        w_obj = space.w_None
     return space.get(w_self, w_obj, w_type)
 
 @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)

From pypy.commits at gmail.com  Mon Jun  5 15:58:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 12:58:04 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: move lto to build option,
 default is False since some gcc versions produce bogus code
Message-ID: <5935b7cc.8388df0a.3a66.e129@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91532:bd5301bc87ef
Date: 2017-06-05 22:54 +0300
http://bitbucket.org/pypy/pypy/changeset/bd5301bc87ef/

Log:	move lto to build option, default is False since some gcc versions
	produce bogus code (grafted from
	c6a3a65d4975769d0767d2d682e4d9741a8042c4)

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -202,6 +202,9 @@
     BoolOption("lldebug0",
                "If true, makes an lldebug0 build", default=False,
                cmdline="--lldebug0"),
+    BoolOption("lto", "enable link time optimization",
+               default=False, cmdline="--lto",
+               requires=[("translation.gcrootfinder", "shadowstack")]),
     StrOption("icon", "Path to the (Windows) icon to use for the executable"),
     StrOption("libname",
               "Windows: name and possibly location of the lib file to create"),
diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py
--- a/rpython/translator/platform/posix.py
+++ b/rpython/translator/platform/posix.py
@@ -134,7 +134,7 @@
             cflags = tuple(self.cflags) + tuple(self.standalone_only)
 
         # xxx check which compilers accept this option or not
-        if not config or config.translation.gcrootfinder != 'asmgcc':
+        if config and config.translation.lto:
             cflags = ('-flto',) + cflags
 
         m = GnuMakefile(path)

From pypy.commits at gmail.com  Mon Jun  5 15:58:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:10 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: tp_descr_get on built-in
 types
Message-ID: <5935b7d2.0bd51c0a.350e1.21b6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r91535:762ce5327b5c
Date: 2017-06-05 22:56 +0300
http://bitbucket.org/pypy/pypy/changeset/762ce5327b5c/

Log:	tp_descr_get on built-in types (grafted from
	4a138a88b24adf195f546cf96aac14a96dde4648)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -820,6 +820,18 @@
         else:
             return
         slot_func = buff_w
+    elif name == 'tp_descr_get':
+        get_fn = w_type.getdictvalue(space, '__get__')
+        if get_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
+            if w_arg1 is None:
+                w_arg1 = space.w_None
+            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        slot_func = slot_tp_descr_get
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -392,6 +392,37 @@
         obj = foo.new()
         assert module.hack_tp_dict(obj) == 2
 
+    def test_tp_descr_get(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_get", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_get == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                return args->ob_type->tp_descr_get(args, NULL,
+                                                   (PyObject *)&PyInt_Type);
+             '''
+             )
+            ])
+        assert module.tp_descr_get(42) is False
+
+        class Y(object):
+            def __get__(self, *args):
+                return 42
+            def unbound_method_example(self):
+                pass
+        assert module.tp_descr_get(Y()) == 42
+        #
+        p = property(lambda self: 42)
+        result = module.tp_descr_get(p)
+        assert result is p
+        #
+        f = lambda x: x + 1
+        ubm = module.tp_descr_get(f)
+        assert type(ubm) is type(Y.unbound_method_example)
+        assert ubm(42) == 43
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 15:58:14 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:14 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: Fix: 'tp_descr_get(self,
 NULL, type)' used to give a real undefined
Message-ID: <5935b7d6.50841c0a.343c3.1589@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.5-5.x
Changeset: r91537:b746b3cc0b2a
Date: 2017-06-05 22:57 +0300
http://bitbucket.org/pypy/pypy/changeset/b746b3cc0b2a/

Log:	Fix: 'tp_descr_get(self, NULL, type)' used to give a real undefined
	value for the second argument if implemented in Python (grafted from
	07257b27db4b6e747d50f24108835f3ed3f62a79)

diff --git a/pypy/module/cpyext/test/test_userslots.py b/pypy/module/cpyext/test/test_userslots.py
--- a/pypy/module/cpyext/test/test_userslots.py
+++ b/pypy/module/cpyext/test/test_userslots.py
@@ -51,7 +51,7 @@
         w_descr = space.appexec([], """():
             class Descr(object):
                 def __get__(self, obj, type):
-                    return 42
+                    return 42 + (obj is None)
                 def __set__(self, obj, value):
                     obj.append('set')
                 def __delete__(self, obj):
@@ -73,6 +73,11 @@
             space, py_descrtype.c_tp_descr_set,
             py_descr, py_obj, None) == 0
         assert space.eq_w(w_obj, space.wrap(['set', 'del']))
+        #
+        # unbound __get__(self, NULL, type)
+        w_res = generic_cpy_call(space, py_descrtype.c_tp_descr_get,
+                                 py_descr, None, space.w_int)
+        assert space.int_w(w_res) == 43
 
 class AppTestUserSlots(AppTestCpythonExtensionBase):
     def test_tp_hash_from_python(self):
diff --git a/pypy/module/cpyext/userslot.py b/pypy/module/cpyext/userslot.py
--- a/pypy/module/cpyext/userslot.py
+++ b/pypy/module/cpyext/userslot.py
@@ -111,6 +111,8 @@
 
 @slot_function([PyObject, PyObject, PyObject], PyObject)
 def slot_tp_descr_get(space, w_self, w_obj, w_type):
+    if w_obj is None:
+        w_obj = space.w_None
     return space.get(w_self, w_obj, w_type)
 
 @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)

From pypy.commits at gmail.com  Mon Jun  5 15:58:18 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:18 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: tp_descr_set on built-in
 types
Message-ID: <5935b7da.4f421c0a.fb900.2efe@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.5-5.x
Changeset: r91539:a39af0be3a22
Date: 2017-06-05 22:57 +0300
http://bitbucket.org/pypy/pypy/changeset/a39af0be3a22/

Log:	tp_descr_set on built-in types (grafted from
	7ca42aeab616fee1898cd0b2bdaef48be7825e3b)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -826,11 +826,32 @@
 
         @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
-        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
-            if w_arg1 is None:
-                w_arg1 = space.w_None
-            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        def slot_tp_descr_get(space, w_self, w_obj, w_value):
+            if w_obj is None:
+                w_obj = space.w_None
+            return space.call_function(get_fn, w_self, w_obj, w_value)
         slot_func = slot_tp_descr_get
+    elif name == 'tp_descr_set':
+        set_fn = w_type.getdictvalue(space, '__set__')
+        delete_fn = w_type.getdictvalue(space, '__delete__')
+        if set_fn is None and delete_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_set(space, w_self, w_obj, w_value):
+            if w_value is not None:
+                if set_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __set__", typedef.name)
+                space.call_function(set_fn, w_self, w_obj, w_value)
+            else:
+                if delete_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __delete__", typedef.name)
+                space.call_function(delete_fn, w_self, w_obj)
+            return 0
+        slot_func = slot_tp_descr_set
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -405,6 +405,42 @@
         assert type(ubm) is type(Y.unbound_method_example)
         assert ubm(42) == 43
 
+    def test_tp_descr_set(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_set", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_set == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                if (args->ob_type->tp_descr_set(args, Py_False, Py_True) != 0)
+                    return NULL;
+                if (args->ob_type->tp_descr_set(args, Py_Ellipsis, NULL) != 0)
+                    return NULL;
+
+                Py_INCREF(Py_True);
+                return Py_True;
+             '''
+             )
+            ])
+        assert module.tp_descr_set(42) is False
+
+        class Y(object):
+            def __set__(self, obj, value):
+                assert obj is False
+                assert value is True
+            def __delete__(self, obj):
+                assert obj is Ellipsis
+        assert module.tp_descr_set(Y()) is True
+        #
+        def pset(obj, value):
+            assert obj is False
+            assert value is True
+        def pdel(obj):
+            assert obj is Ellipsis
+        p = property(lambda: "never used", pset, pdel)
+        assert module.tp_descr_set(p) is True
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 15:58:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:12 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: tp_descr_set on built-in
 types
Message-ID: <5935b7d4.b885df0a.cad2b.c924@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy2.7-5.x
Changeset: r91536:57722d241bf1
Date: 2017-06-05 22:56 +0300
http://bitbucket.org/pypy/pypy/changeset/57722d241bf1/

Log:	tp_descr_set on built-in types (grafted from
	7ca42aeab616fee1898cd0b2bdaef48be7825e3b)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -827,11 +827,32 @@
 
         @slot_function([PyObject, PyObject, PyObject], PyObject)
         @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
-        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
-            if w_arg1 is None:
-                w_arg1 = space.w_None
-            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        def slot_tp_descr_get(space, w_self, w_obj, w_value):
+            if w_obj is None:
+                w_obj = space.w_None
+            return space.call_function(get_fn, w_self, w_obj, w_value)
         slot_func = slot_tp_descr_get
+    elif name == 'tp_descr_set':
+        set_fn = w_type.getdictvalue(space, '__set__')
+        delete_fn = w_type.getdictvalue(space, '__delete__')
+        if set_fn is None and delete_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], rffi.INT_real, error=-1)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_set(space, w_self, w_obj, w_value):
+            if w_value is not None:
+                if set_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __set__", typedef.name)
+                space.call_function(set_fn, w_self, w_obj, w_value)
+            else:
+                if delete_fn is None:
+                    raise oefmt(space.w_TypeError,
+                                "%s object has no __delete__", typedef.name)
+                space.call_function(delete_fn, w_self, w_obj)
+            return 0
+        slot_func = slot_tp_descr_set
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -423,6 +423,42 @@
         assert type(ubm) is type(Y.unbound_method_example)
         assert ubm(42) == 43
 
+    def test_tp_descr_set(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_set", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_set == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                if (args->ob_type->tp_descr_set(args, Py_False, Py_True) != 0)
+                    return NULL;
+                if (args->ob_type->tp_descr_set(args, Py_Ellipsis, NULL) != 0)
+                    return NULL;
+
+                Py_INCREF(Py_True);
+                return Py_True;
+             '''
+             )
+            ])
+        assert module.tp_descr_set(42) is False
+
+        class Y(object):
+            def __set__(self, obj, value):
+                assert obj is False
+                assert value is True
+            def __delete__(self, obj):
+                assert obj is Ellipsis
+        assert module.tp_descr_set(Y()) is True
+        #
+        def pset(obj, value):
+            assert obj is False
+            assert value is True
+        def pdel(obj):
+            assert obj is Ellipsis
+        p = property(lambda: "never used", pset, pdel)
+        assert module.tp_descr_set(p) is True
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 15:58:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 12:58:16 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: tp_descr_get on built-in
 types
Message-ID: <5935b7d8.84ae1c0a.7dd44.a269@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: release-pypy3.5-5.x
Changeset: r91538:932fcb68bd70
Date: 2017-06-05 22:57 +0300
http://bitbucket.org/pypy/pypy/changeset/932fcb68bd70/

Log:	tp_descr_get on built-in types (grafted from
	4a138a88b24adf195f546cf96aac14a96dde4648)

diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py
--- a/pypy/module/cpyext/slotdefs.py
+++ b/pypy/module/cpyext/slotdefs.py
@@ -819,6 +819,18 @@
         else:
             return
         slot_func = buff_w
+    elif name == 'tp_descr_get':
+        get_fn = w_type.getdictvalue(space, '__get__')
+        if get_fn is None:
+            return
+
+        @slot_function([PyObject, PyObject, PyObject], PyObject)
+        @func_renamer("cpyext_%s_%s" % (name.replace('.', '_'), typedef.name))
+        def slot_tp_descr_get(space, w_self, w_arg1, w_arg2):
+            if w_arg1 is None:
+                w_arg1 = space.w_None
+            return space.call_function(get_fn, w_self, w_arg1, w_arg2)
+        slot_func = slot_tp_descr_get
     else:
         # missing: tp_as_number.nb_nonzero, tp_as_number.nb_coerce
         # tp_as_sequence.c_sq_contains, tp_as_sequence.c_sq_length
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -374,6 +374,37 @@
         obj = foo.new()
         assert module.hack_tp_dict(obj) == 2
 
+    def test_tp_descr_get(self):
+        module = self.import_extension('foo', [
+           ("tp_descr_get", "METH_O",
+            '''
+                if (args->ob_type->tp_descr_get == NULL) {
+                    Py_INCREF(Py_False);
+                    return Py_False;
+                }
+                return args->ob_type->tp_descr_get(args, NULL,
+                                                   (PyObject *)&PyInt_Type);
+             '''
+             )
+            ])
+        assert module.tp_descr_get(42) is False
+
+        class Y(object):
+            def __get__(self, *args):
+                return 42
+            def unbound_method_example(self):
+                pass
+        assert module.tp_descr_get(Y()) == 42
+        #
+        p = property(lambda self: 42)
+        result = module.tp_descr_get(p)
+        assert result is p
+        #
+        f = lambda x: x + 1
+        ubm = module.tp_descr_get(f)
+        assert type(ubm) is type(Y.unbound_method_example)
+        assert ubm(42) == 43
+
 
 class TestTypes(BaseApiTest):
     def test_type_attributes(self, space, api):

From pypy.commits at gmail.com  Mon Jun  5 16:26:08 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 13:26:08 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: implement stop/start_sampling and
 copy over new implementation from vmprof-python.git
 6142531a47d6c294b1fd096aec7b9f3df6b8dfb5
Message-ID: <5935be60.568bdf0a.161cb.36f1@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91540:a4f077ba651c
Date: 2017-06-05 16:25 -0400
http://bitbucket.org/pypy/pypy/changeset/a4f077ba651c/

Log:	implement stop/start_sampling and copy over new implementation from
	vmprof-python.git 6142531a47d6c294b1fd096aec7b9f3df6b8dfb5

diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -14,6 +14,9 @@
         'write_all_code_objects': 'interp_vmprof.write_all_code_objects',
         'is_enabled': 'interp_vmprof.is_enabled',
         'get_profile_path': 'interp_vmprof.get_profile_path',
+        'stop_sampling': 'interp_vmprof.stop_sampling',
+        'start_sampling': 'interp_vmprof.start_sampling',
+
         'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
     }
 
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -96,3 +96,10 @@
         # Indicates an error! Assume platform does not implement the function call
         raise oefmt(space.w_NotImplementedError, "platform not implemented")
     return space.newtext(path)
+
+def stop_sampling(space):
+    return space.newint(rvmprof.stop_sampling(space))
+
+def start_sampling(space):
+    rvmprof.start_sampling(space)
+    return space.w_None
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -3,7 +3,7 @@
 from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME
 from rpython.rlib.rvmprof.rvmprof import _was_registered
 from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 #
 # See README.txt.
@@ -56,3 +56,9 @@
 
     return None
 
+def stop_sampling(space):
+    fd = _get_vmprof().cintf.stop_sampling()
+    return rffi.cast(lltype.Signed, fd)
+
+def start_sampling(space):
+    _get_vmprof().cintf.start_sampling()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -110,6 +110,12 @@
     vmprof_get_profile_path = rffi.llexternal("vmprof_get_profile_path", [rffi.CCHARP, lltype.Signed],
                                               lltype.Signed, compilation_info=eci,
                                               _nowrapper=True)
+    vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
+                                              rffi.INT, compilation_info=eci,
+                                              _nowrapper=True)
+    vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
+                                              lltype.Void, compilation_info=eci,
+                                              _nowrapper=True)
 
     return CInterface(locals())
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -30,8 +30,18 @@
 }
 #endif
 
-
 long vmprof_get_profile_path(const char * buffer, long size)
 {
     return vmp_fd_to_path(vmp_profile_fileno(), buffer, size);
 }
+
+int vmprof_stop_sampling(void)
+{
+    vmprof_ignore_signals(1);
+    return vmp_profile_fileno();
+}
+
+void vmprof_start_sampling(void)
+{
+    vmprof_ignore_signals(0);
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -38,6 +38,8 @@
 RPY_EXTERN void vmprof_stack_free(void*);
 RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
 RPY_EXTERN long vmprof_get_profile_path(const char *, long);
+RPY_EXTERN int vmprof_stop_sampling(void);
+RPY_EXTERN void vmprof_start_sampling(void);
 
 long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
                                       intptr_t addr, int max_depth);
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -13,7 +13,6 @@
 static volatile int is_enabled = 0;
 static destructor Original_code_dealloc = 0;
 static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
-void vmp_scan_profile(int fileno, int dump_native, void *all_code_uids);
 
 #if VMPROF_UNIX
 #include "trampoline.h"
@@ -82,60 +81,39 @@
     return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000);
 }
 
-static int _look_for_code_object(PyObject *o, void *all_codes)
+static int _look_for_code_object(PyObject *o, void * param)
 {
-    if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) {
-        Py_ssize_t i;
+    PyObject * all_codes = (PyObject*)((void**)param)[0];
+    PyObject * seen_codes = (PyObject*)((void**)param)[1];
+    if (PyCode_Check(o) && !PySet_Contains(all_codes, o)) {
         PyCodeObject *co = (PyCodeObject *)o;
-        if (emit_code_object(co) < 0)
-            return -1;
-        if (PySet_Add((PyObject *)all_codes, o) < 0)
-            return -1;
+        PyObject * id = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co));
+        if (PySet_Contains(seen_codes, id)) {
+            // only emit if the code id has been seen!
+            if (emit_code_object(co) < 0)
+                return -1;
+            if (PySet_Add(all_codes, o) < 0)
+                return -1;
+        }
 
         /* as a special case, recursively look for and add code
            objects found in the co_consts.  The problem is that code
            objects are not created as GC-aware in CPython, so we need
            to hack like this to hope to find most of them. 
         */
-        i = PyTuple_Size(co->co_consts);
+        int i = PyTuple_Size(co->co_consts);
         while (i > 0) {
             --i;
             if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
-                                      all_codes) < 0)
+                                      param) < 0)
                 return -1;
         }
     }
     return 0;
 }
 
-static int _look_for_code_object_seen(PyObject *o, void *all_codes)
-{
-    if (PyCode_Check(o) && PySet_GET_SIZE(all_codes)) {
-        Py_ssize_t i;
-        PyCodeObject *co = (PyCodeObject *)o;
-        PyObject *uid_co = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co));
-        int check = PySet_Discard(all_codes, uid_co);
-
-        Py_CLEAR(uid_co);
-
-        if (check < 0)
-            return -1;
-
-        if (check && emit_code_object(co) < 0)
-            return -1;
-
-        i = PyTuple_Size(co->co_consts);
-        while (i > 0) {
-            --i;
-            if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
-                                      all_codes) < 0)
-                return -1;
-        }
-    }
-    return 0;
-}
-
-static void emit_all_code_objects(void)
+static
+void emit_all_code_objects(PyObject * seen_code_ids)
 {
     PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
     Py_ssize_t i, size;
@@ -154,11 +132,15 @@
     if (all_codes == NULL)
         goto error;
 
+    void * param[2];
+    param[0] = all_codes;
+    param[1] = seen_code_ids;
+
     size = PyList_GET_SIZE(lst);
     for (i = 0; i < size; i++) {
         PyObject *o = PyList_GET_ITEM(lst, i);
         if (o->ob_type->tp_traverse &&
-            o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes)
+            o->ob_type->tp_traverse(o, _look_for_code_object, (void*)param)
                 < 0)
             goto error;
     }
@@ -169,91 +151,6 @@
     Py_XDECREF(gc_module);
 }
 
-static int add_code_addr(void *all_code_uids, void *addr)
-{
-    PyObject *co_uid = PyLong_FromVoidPtr(addr);
-    int check = PySet_Add((PyObject*) all_code_uids, co_uid);
-    Py_CLEAR(co_uid);
-    return check;
-}
-
-static void emit_all_code_objects_seen(int fileno)
-{
-    PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
-    Py_ssize_t i, size;
-
-    gc_module = PyImport_ImportModuleNoBlock("gc");
-    if (gc_module == NULL)
-        goto error;
-
-    lst = PyObject_CallMethod(gc_module, "get_objects", "");
-    if (lst == NULL || !PyList_Check(lst))
-        goto error;
-
-    all_codes = PySet_New(NULL);
-    if (all_codes == NULL)
-        goto error;
-
-    // fill up all_codes with every code object found in the profile
-    vmp_scan_profile(fileno, 0, all_codes);
-
-    // intersect the list with the set and dump only the code objects
-    // found in the set!
-    size = PyList_GET_SIZE(lst);
-    for (i = 0; i < size; i++) {
-        PyObject *o = PyList_GET_ITEM(lst, i);
-        if (o->ob_type->tp_traverse &&
-                o->ob_type->tp_traverse(o, _look_for_code_object_seen, (void *) all_codes)
-            < 0)
-            goto error;
-    }
-
-    error:
-    Py_XDECREF(all_codes);
-    Py_XDECREF(lst);
-    Py_XDECREF(gc_module);
-}
-
-static int emit_all_code_objects_helper(int only_needed, int disable_vmprof)
-{
-    int fd = vmp_profile_fileno();
-
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
-        return -1;
-    }
-
-#if VMPROF_UNIX
-    if ((read(fd, NULL, 0) != 0) && (only_needed != 0)) {
-        PyErr_SetString(PyExc_ValueError,
-                        "file descriptor must be readable to save only needed code objects");
-        return -1;
-    }
-#else
-    if (only_needed) {
-        PyErr_SetString(PyExc_ValueError,
-                        "saving only needed code objects is not supported for windows");
-        return -1;
-    }
-#endif
-
-    if (disable_vmprof) {
-        is_enabled = 0;
-        vmprof_ignore_signals(1);
-    }
-
-#if VMPROF_UNIX
-    if (only_needed)
-        emit_all_code_objects_seen(fd);
-    else
-        emit_all_code_objects();
-#else
-    emit_all_code_objects();
-#endif
-
-    return 0;
-}
-
 static void cpyprof_code_dealloc(PyObject *co)
 {
     if (is_enabled) {
@@ -283,7 +180,7 @@
     }
 
     if ((read(fd, NULL, 0) != 0) && (native != 0)) {
-        PyErr_SetString(PyExc_ValueError, "file descriptor must be readable for native profiling");
+        PyErr_SetString(PyExc_ValueError, "file descriptor must be readable");
         return NULL;
     }
 
@@ -330,21 +227,15 @@
 }
 
 static PyObject *
-disable_vmprof(PyObject *module, PyObject *args)
+disable_vmprof(PyObject *module, PyObject *noargs)
 {
-    int only_needed = 0;
-
-    if (!PyArg_ParseTuple(args, "|i", &only_needed))
-        return NULL;
-
-    if (emit_all_code_objects_helper(only_needed, 1))
-        return NULL;
-
     if (vmprof_disable() < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
 
+    is_enabled = 0;
+
     if (PyErr_Occurred())
         return NULL;
 
@@ -352,23 +243,18 @@
 }
 
 static PyObject *
-write_all_code_objects(PyObject *module, PyObject *args)
+write_all_code_objects(PyObject *module, PyObject * seen_code_ids)
 {
-    int only_needed = 0;
-
-    if (!PyArg_ParseTuple(args, "|i", &only_needed))
-        return NULL;
-
-    if (emit_all_code_objects_helper(only_needed, 0))
-        return NULL;
+    // assumptions: signals must be disabled (see stop_sampling)
+    emit_all_code_objects(seen_code_ids);
 
     if (PyErr_Occurred())
         return NULL;
-
     Py_RETURN_NONE;
 }
 
 
+
 static PyObject *
 sample_stack_now(PyObject *module, PyObject * args)
 {
@@ -447,7 +333,6 @@
     if (o_srcfile == NULL) goto error;
     //
     return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
-
 error:
     Py_XDECREF(o_name);
     Py_XDECREF(o_lineno);
@@ -457,6 +342,20 @@
 }
 #endif
 
+static PyObject *
+stop_sampling(PyObject *module, PyObject *noargs)
+{
+    vmprof_ignore_signals(1);
+    return PyLong_NEW(vmp_profile_fileno());
+}
+
+static PyObject *
+start_sampling(PyObject *module, PyObject *noargs)
+{
+    vmprof_ignore_signals(0);
+    Py_RETURN_NONE;
+}
+
 #ifdef VMPROF_UNIX
 static PyObject * vmp_get_profile_path(PyObject *module, PyObject *noargs) {
     PyObject * o;
@@ -523,23 +422,30 @@
 }
 #endif
 
-
 static PyMethodDef VMProfMethods[] = {
     {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
-    {"disable", disable_vmprof, METH_VARARGS, "Disable profiling."},
-    {"write_all_code_objects", write_all_code_objects, METH_VARARGS,
-     "Write eagerly all the IDs of code objects"},
-    {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"},
-    {"is_enabled", vmp_is_enabled, METH_NOARGS, "Indicates if vmprof is currently sampling."},
+    {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."},
+    {"write_all_code_objects", write_all_code_objects, METH_O,
+        "Write eagerly all the IDs of code objects"},
+    {"sample_stack_now", sample_stack_now, METH_VARARGS,
+        "Sample the stack now"},
+    {"is_enabled", vmp_is_enabled, METH_NOARGS,
+        "Indicates if vmprof is currently sampling."},
+    {"stop_sampling", stop_sampling, METH_NOARGS,
+        "Blocks signals to occur and returns the file descriptor"},
+    {"start_sampling", start_sampling, METH_NOARGS,
+        "Unblocks vmprof signals. After compeltion vmprof will sample again"},
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
+    {"resolve_addr", resolve_addr, METH_VARARGS,
+        "Returns the name of the given address"},
 #endif
 #ifdef VMPROF_UNIX
-    {"get_profile_path", vmp_get_profile_path, METH_NOARGS, "Profile path the profiler logs to."},
+    {"get_profile_path", vmp_get_profile_path, METH_NOARGS,
+        "Profile path the profiler logs to."},
     {"insert_real_time_thread", insert_real_time_thread, METH_NOARGS,
-     "Insert a thread into the real time profiling list."},
+        "Insert a thread into the real time profiling list."},
     {"remove_real_time_thread", remove_real_time_thread, METH_NOARGS,
-     "Remove a thread from the real time profiling list."},
+        "Remove a thread from the real time profiling list."},
 #endif
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c
--- a/rpython/rlib/rvmprof/src/shared/compat.c
+++ b/rpython/rlib/rvmprof/src/shared/compat.c
@@ -87,7 +87,9 @@
     }
     buf.tv_sec = tv.tv_sec;
     buf.tv_usec = tv.tv_usec;
-    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
+    // IF we really support time zones:
+    // use a cross platform datetime library that outputs iso8601 strings
+    // strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
 
     buffer[0] = marker;
     (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
@@ -128,8 +130,9 @@
     buf.tv_usec = (system_time.wMilliseconds * 1000);
 
     // time zone not implemented on windows
+    // IF we really support time zones:
+    // use a cross platform datetime library that outputs iso8601 strings
     memset(((char*)buffer)+__SIZE-8, 0, 8);
-    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
 
     buffer[0] = marker;
     (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -11,7 +11,6 @@
 
 #include <assert.h>
 #include <dlfcn.h>
-#include <errno.h>
 
 #if defined(VMPROF_LINUX)
 #include <link.h>
@@ -266,272 +265,3 @@
 #endif
     return 0;
 }
-
-#define WORD_SIZE sizeof(long)
-#define ADDR_SIZE sizeof(void*)
-#define MAXLEN 1024
-
-void _dump_native_symbol(int fileno, void * addr, char * sym, int linenumber, char * filename) {
-    char natsym[64];
-    off_t pos_before;
-    struct str {
-        void * addr;
-        // NOTE windows, not supported yet would be a problem for 64 bit
-        // hint: alignment
-        long size;
-        char str[1024];
-    } s;
-    fsync(fileno);
-    pos_before = lseek(fileno, 0, SEEK_CUR);
-    lseek(fileno, 0, SEEK_END);
-
-    s.addr = addr;
-    /* must mach '<lang>:<name>:<line>:<file>'
-     * 'n' has been chosen as lang here, because the symbol
-     * can be generated from several languages (e.g. C, C++, ...)
-     */
-    // MARKER_NATIVE_SYMBOLS is \x08
-    write(fileno, "\x08", 1);
-    if (sym == NULL || sym[0] == '\x00') {
-        snprintf(natsym, 64, "<native symbol %p>", addr);
-        sym = natsym;
-    }
-    if (filename != NULL) {
-        s.size = snprintf(s.str, 1024, "n:%s:%d:%s", sym, linenumber, filename);
-    } else {
-        s.size = snprintf(s.str, 1024, "n:%s:%d:-", sym, linenumber);
-    }
-    write(fileno, &s, sizeof(void*)+sizeof(long)+s.size);
-
-    lseek(fileno, pos_before, SEEK_SET);
-}
-
-static
-int cannot_read_profile = 0;
-
-ssize_t read_exactly(int fileno, void * buf, ssize_t size) {
-    assert(size >= 0 && "size parameter must be positive");
-
-    ssize_t r = 0;
-    if ((r = read(fileno, buf, (size_t)size)) == size) {
-        return r;
-    }
-
-    if (r == -1) {
-        while (errno == EINTR) {
-            if ((r = read(fileno, buf, (size_t)size)) == size) {
-                return r;
-            }
-        }
-    }
-
-    fprintf(stderr, "unhandled error in read_exactly. cannot proceed! could not read %d bytes", size);
-    cannot_read_profile = 1;
-    return -1;
-}
-
-int _skip_string(int fileno)
-{
-    long chars;
-    ssize_t count = read_exactly(fileno, &chars, sizeof(long));
-    //LOG("reading string of %d chars\n", chars);
-    if (count <= 0) {
-        return 1;
-    }
-    lseek(fileno, chars, SEEK_CUR);
-
-    return 0;
-}
-
-int _skip_header(int fileno, int * version, int * flags)
-{
-    unsigned char r[4];
-    (void)read_exactly(fileno, r, 4);
-    unsigned char count = r[3];
-    *version = (r[0] & 0xff) << 8 | (r[1] & 0xff);
-    *flags = r[2];
-    lseek(fileno, (int)count, SEEK_CUR);
-    return 0;
-}
-
-long _read_word(int fileno)
-{
-    long w;
-    (void)read_exactly(fileno, &w, WORD_SIZE);
-    return w;
-}
-
-void * _read_addr(int fileno)
-{
-    void * a;
-    (void)read_exactly(fileno, &a, ADDR_SIZE);
-    return a;
-}
-
-int _skip_word(int fileno)
-{
-    lseek(fileno, WORD_SIZE, SEEK_CUR);
-    return 0;
-}
-
-int _skip_addr(int fileno)
-{
-    lseek(fileno, ADDR_SIZE, SEEK_CUR);
-    return 0;
-}
-
-int _skip_time_and_zone(int fileno)
-{
-    lseek(fileno, sizeof(int64_t)*2 + 8, SEEK_CUR);
-    return 0;
-}
-
-KHASH_MAP_INIT_INT64(ptr, int)
-
-void vmp_scan_profile(int fileno, int dump_nat_sym, void *all_code_uids)
-{
-    off_t orig_pos, cur_pos;
-    char marker;
-    ssize_t count;
-    int version;
-    int flags;
-    int memory = 0, lines = 0, native = 0;
-    fsync(fileno);
-    orig_pos = lseek(fileno, 0, SEEK_CUR);
-
-    khash_t(ptr) * nat_syms = kh_init(ptr);
-    khiter_t it;
-
-    lseek(fileno, 5*WORD_SIZE, SEEK_SET);
-
-    while (1) {
-        if (cannot_read_profile == 1) {
-            cannot_read_profile = 0;
-            break;
-        }
-        count = read_exactly(fileno, &marker, 1);
-        if (count <= 0) {
-            break;
-        }
-        cur_pos = lseek(fileno, 0, SEEK_CUR);
-        LOG("posss 0x%llx %d\n", cur_pos-1, cur_pos-1);
-        switch (marker) {
-            case MARKER_HEADER: {
-                LOG("header 0x%llx\n", cur_pos);
-                if (_skip_header(fileno, &version, &flags) != 0) {
-                    kh_destroy(ptr, nat_syms);
-                    return;
-                }
-                memory = (flags & PROFILE_MEMORY) != 0;
-                native = (flags & PROFILE_NATIVE) != 0;
-                lines = (flags & PROFILE_LINES) != 0;
-                if (!native && dump_nat_sym) {
-                    lseek(fileno, 0, SEEK_END);
-                    kh_destroy(ptr, nat_syms);
-                    return;
-                }
-                break;
-            } case MARKER_META: {
-                LOG("meta 0x%llx\n", cur_pos);
-                if (_skip_string(fileno) != 0) { return; }
-                if (_skip_string(fileno) != 0) { return; }
-                break;
-            } case MARKER_TIME_N_ZONE:
-              case MARKER_TRAILER: {
-                LOG("tnz or trailer 0x%llx\n", cur_pos);
-                if (_skip_time_and_zone(fileno) != 0) { return; }
-                break;
-            } case MARKER_VIRTUAL_IP:
-              case MARKER_NATIVE_SYMBOLS: {
-                //LOG("virtip 0x%llx\n", cur_pos);
-                if (_skip_addr(fileno) != 0) { return; }
-                if (_skip_string(fileno) != 0) { return; }
-                break;
-            } case MARKER_STACKTRACE: {
-                long trace_count = _read_word(fileno);
-                long depth = _read_word(fileno);
-                long i;
-
-                LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
-
-#ifdef RPYTHON_VMPROF
-                for (i = depth/2-1; i >= 0; i--) {
-                    long kind = (long)_read_addr(fileno);
-                    void * addr = _read_addr(fileno);
-                    if (kind == VMPROF_NATIVE_TAG) {
-#else
-                for (i = 0; i < depth; i++) {
-                    void * addr = _read_addr(fileno);
-                    if (lines && i % 2 == 0) {
-                        continue;
-                    }
-                    if (((intptr_t)addr & 0x1) == 1) {
-#endif
-                        /* dump the native symbol to disk */
-                        if (dump_nat_sym) {
-                            LOG("found kind %p\n", addr);
-
-                            // if the address has already been dumped,
-                            // do not log it again!
-                            it = kh_get(ptr, nat_syms, (khint64_t)addr);
-                            if (it == kh_end(nat_syms)) {
-                                char name[MAXLEN];
-                                char srcfile[MAXLEN];
-                                name[0] = 0;
-                                srcfile[0] = 0;
-                                int lineno = 0;
-                                if (vmp_resolve_addr(addr, name, MAXLEN, &lineno, srcfile, MAXLEN) == 0) {
-                                    LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno);
-                                    _dump_native_symbol(fileno, addr, name, lineno, srcfile);
-                                    int ret;
-                                    it = kh_put(ptr, nat_syms, (khint64_t)addr, &ret);
-                                    kh_value(nat_syms, it) = 1;
-                                }
-                            }
-                        }
-#ifdef RPYTHON_VMPROF
-                    }
-#else
-                    } else {
-                        // cpython adds all addresses into a set to get the intersection
-                        // of all gc known code addresses
-                        if (all_code_uids != NULL) {
-                            PyObject *co_uid = PyLong_FromVoidPtr(addr);
-                            int check = PySet_Add(all_code_uids, co_uid);
-                            Py_CLEAR(co_uid);
-                        }
-                    }
-#endif
-                }
-                LOG("passed  memory %d \n", memory);
-
-                if (version >= VERSION_THREAD_ID) {
-                    if (_skip_addr(fileno) != 0) { return; } // thread id
-                }
-                if (memory) {
-                    if (_skip_addr(fileno) != 0) { return; } // profile memory
-                }
-
-                break;
-            } default: {
-                fprintf(stderr, "unknown marker 0x%x\n", marker);
-                lseek(fileno, 0, SEEK_END);
-                kh_destroy(ptr, nat_syms);
-                return;
-            }
-        }
-
-        cur_pos = lseek(fileno, 0, SEEK_CUR);
-        if (cur_pos >= orig_pos) {
-            break;
-        }
-    }
-
-    kh_destroy(ptr, nat_syms);
-    lseek(fileno, 0, SEEK_END);
-}
-
-void dump_native_symbols(int fileno)
-{
-    vmp_scan_profile(fileno, 1, NULL);
-}
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.h b/rpython/rlib/rvmprof/src/shared/symboltable.h
--- a/rpython/rlib/rvmprof/src/shared/symboltable.h
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.h
@@ -2,22 +2,5 @@
 
 #define _GNU_SOURCE 1
 
-/**
- * Extract all the known symbols from the current process and
- * log them to the file descriptor. To read them see binary.py funcs:
- *
- * # encoded as a mapping
- * addr = read_word(fd); name = read_string(fd)
- *
- * A) It is not allowed to have two addresses (virtual ones are only valid
- * in the curent process) in this mapping to point to several symbols.
- * B) No duplicates are logged
- *
- * Deprecated, do not use
- */
-void dump_all_known_symbols(int fd);
-
-void dump_native_symbols(int fd);
-
 int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno,
                       char * srcfile, int srcfile_len);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -39,7 +39,7 @@
 static inline ssize_t search_thread(pthread_t tid, ssize_t i) {
     if (i < 0)
         i = 0;
-    while (i < thread_count) {
+    while ((size_t)i < thread_count) {
         if (pthread_equal(threads[i], tid))
             return i;
         i++;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -166,7 +166,7 @@
     mythread_id = PyThread_get_thread_ident();
     istate = PyInterpreterState_Head();
     if (istate == NULL) {
-        fprintf(stderr, "WARNING: interp state head is null (for thread id %d)\n", mythread_id);
+        fprintf(stderr, "WARNING: interp state head is null (for thread id %ld)\n", mythread_id);
         return NULL;
     }
     // fish fish fish, it will NOT lock the keymutex in pythread
@@ -180,7 +180,7 @@
     } while ((istate = PyInterpreterState_Next(istate)) != NULL);
 
     // uh? not found?
-    fprintf(stderr, "WARNING: cannot find thread state (for thread id %d), sample will be thrown away\n", mythread_id);
+    fprintf(stderr, "WARNING: cannot find thread state (for thread id %ld), sample will be thrown away\n", mythread_id);
     return NULL;
 }
 #endif
@@ -462,13 +462,9 @@
 {
     int fileno = vmp_profile_fileno();
     fsync(fileno);
-    dump_native_symbols(fileno);
-
     (void)vmp_write_time_now(MARKER_TRAILER);
-
     teardown_rss();
 
-
     /* don't close() the file descriptor from here */
     vmp_set_profile_fileno(-1);
     return 0;
@@ -483,13 +479,16 @@
     disable_cpyprof();
 #endif
 
-    if (remove_sigprof_timer() == -1)
+    if (remove_sigprof_timer() == -1) {
         return -1;
-    if (remove_sigprof_handler() == -1)
+    }
+    if (remove_sigprof_handler() == -1) {
         return -1;
+    }
 #ifdef VMPROF_UNIX
-    if ((signal_type == SIGALRM) && remove_threads() == -1)
+    if ((signal_type == SIGALRM) && remove_threads() == -1) {
         return -1;
+    }
 #endif
     flush_codes();
     if (shutdown_concurrent_bufs(vmp_profile_fileno()) < 0)

From pypy.commits at gmail.com  Mon Jun  5 16:41:31 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 13:41:31 -0700 (PDT)
Subject: [pypy-commit] pypy default: update contributors
Message-ID: <5935c1fb.adaedf0a.caf39.0223@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91541:0a451f1495e9
Date: 2017-06-05 23:40 +0300
http://bitbucket.org/pypy/pypy/changeset/0a451f1495e9/

Log:	update contributors

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -101,28 +101,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -149,13 +149,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -164,11 +164,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -183,6 +183,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -220,12 +221,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -264,7 +267,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -303,6 +305,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -340,11 +343,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -68,28 +68,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -116,13 +116,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -131,11 +131,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -150,6 +150,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -187,12 +188,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -231,7 +234,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -270,6 +272,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -307,11 +310,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Richard Plangger':['planrich'],
     'Michael Hudson': ['mwh'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
     "Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
-    'Stanislaw Halik':['w31rd0'],
+    'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
     'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
     'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
     'Jasper Schulz':['Jasper.Schulz', 'jbs'],
     'Aaron Gallagher':['"Aaron Gallagher'],
     'Yasir Suhail':['yasirs'],
-    'Squeaky', ['squeaky'],
+    'Squeaky': ['squeaky'],
+    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     }
 
 alias_map = {}

From pypy.commits at gmail.com  Mon Jun  5 16:41:36 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 13:41:36 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy2.7-5.x: update contributors
Message-ID: <5935c200.899adf0a.997e.b389@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy2.7-5.x
Changeset: r91543:c925e7381036
Date: 2017-06-05 23:40 +0300
http://bitbucket.org/pypy/pypy/changeset/c925e7381036/

Log:	update contributors (grafted from
	0a451f1495e939e97099b714ce428509d96523ca)

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -101,28 +101,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -149,13 +149,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -164,11 +164,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -183,6 +183,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -220,12 +221,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -264,7 +267,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -303,6 +305,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -340,11 +343,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -68,28 +68,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -116,13 +116,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -131,11 +131,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -150,6 +150,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -187,12 +188,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -231,7 +234,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -270,6 +272,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -307,11 +310,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Richard Plangger':['planrich'],
     'Michael Hudson': ['mwh'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
     "Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
-    'Stanislaw Halik':['w31rd0'],
+    'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
     'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
     'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
     'Jasper Schulz':['Jasper.Schulz', 'jbs'],
     'Aaron Gallagher':['"Aaron Gallagher'],
     'Yasir Suhail':['yasirs'],
-    'Squeaky', ['squeaky'],
+    'Squeaky': ['squeaky'],
+    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     }
 
 alias_map = {}

From pypy.commits at gmail.com  Mon Jun  5 16:41:33 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 13:41:33 -0700 (PDT)
Subject: [pypy-commit] pypy release-pypy3.5-5.x: update contributors
Message-ID: <5935c1fd.38a6df0a.12763.17c0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: release-pypy3.5-5.x
Changeset: r91542:a37ecfe5f142
Date: 2017-06-05 23:40 +0300
http://bitbucket.org/pypy/pypy/changeset/a37ecfe5f142/

Log:	update contributors (grafted from
	0a451f1495e939e97099b714ce428509d96523ca)

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -101,28 +101,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -149,13 +149,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -164,11 +164,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -183,6 +183,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -220,12 +221,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -264,7 +267,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -303,6 +305,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -340,11 +343,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -68,28 +68,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -116,13 +116,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -131,11 +131,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -150,6 +150,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -187,12 +188,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -231,7 +234,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -270,6 +272,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -307,11 +310,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Richard Plangger':['planrich'],
     'Michael Hudson': ['mwh'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
     "Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
-    'Stanislaw Halik':['w31rd0'],
+    'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
     'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
     'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
     'Jasper Schulz':['Jasper.Schulz', 'jbs'],
     'Aaron Gallagher':['"Aaron Gallagher'],
     'Yasir Suhail':['yasirs'],
-    'Squeaky', ['squeaky'],
+    'Squeaky': ['squeaky'],
+    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     }
 
 alias_map = {}

From pypy.commits at gmail.com  Mon Jun  5 16:49:40 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 13:49:40 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak release document
Message-ID: <5935c3e4.4c3a1c0a.f0b46.ff18@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91544:fc6cbab9b7aa
Date: 2017-06-05 23:45 +0300
http://bitbucket.org/pypy/pypy/changeset/fc6cbab9b7aa/

Log:	tweak release document

diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -5,8 +5,8 @@
 ++++++++++++++
 
 We try to create a stable release a few times a year. These are released on
-a branch named like release-2.x or release-4.x, and each release is tagged,
-for instance release-4.0.1. 
+a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
+release is tagged, for instance release-pypy3.5-v4.0.1. 
 
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
@@ -33,7 +33,7 @@
 * If needed, make a release branch
 * Bump the
   pypy version number in module/sys/version.py and in
-  module/cpyext/include/patchlevel.h and . The branch
+  module/cpyext/include/patchlevel.h and in doc/conf.py. The branch
   will capture the revision number of this change for the release.
 
   Some of the next updates may be done before or after branching; make

From pypy.commits at gmail.com  Mon Jun  5 16:52:00 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Mon, 05 Jun 2017 13:52:00 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: more fixes for mvsc (copied over
 from vmprof-python)
Message-ID: <5935c470.aba0df0a.5adc2.51d6@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91545:32433b2340ea
Date: 2017-06-05 16:51 -0400
http://bitbucket.org/pypy/pypy/changeset/32433b2340ea/

Log:	more fixes for mvsc (copied over from vmprof-python)

diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -9,7 +9,6 @@
 # See README.txt.
 #
 
-
 #vmprof_execute_code(): implemented directly in rvmprof.py
 
 def register_code_object_class(CodeClass, full_name_func):
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -83,8 +83,11 @@
 
 static int _look_for_code_object(PyObject *o, void * param)
 {
-    PyObject * all_codes = (PyObject*)((void**)param)[0];
-    PyObject * seen_codes = (PyObject*)((void**)param)[1];
+    int i;
+    PyObject * all_codes, * seen_codes;
+
+    all_codes = (PyObject*)((void**)param)[0];
+    seen_codes = (PyObject*)((void**)param)[1];
     if (PyCode_Check(o) && !PySet_Contains(all_codes, o)) {
         PyCodeObject *co = (PyCodeObject *)o;
         PyObject * id = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co));
@@ -101,7 +104,7 @@
            objects are not created as GC-aware in CPython, so we need
            to hack like this to hope to find most of them. 
         */
-        int i = PyTuple_Size(co->co_consts);
+        i = PyTuple_Size(co->co_consts);
         while (i > 0) {
             --i;
             if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
@@ -117,6 +120,7 @@
 {
     PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
     Py_ssize_t i, size;
+    void * param[2];
 
     gc_module = PyImport_ImportModuleNoBlock("gc");
     if (gc_module == NULL)
@@ -132,7 +136,6 @@
     if (all_codes == NULL)
         goto error;
 
-    void * param[2];
     param[0] = all_codes;
     param[1] = seen_code_ids;
 

From pypy.commits at gmail.com  Mon Jun  5 17:04:27 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 05 Jun 2017 14:04:27 -0700 (PDT)
Subject: [pypy-commit] pypy default: tweak release notice
Message-ID: <5935c75b.ae86df0a.1c9c5.5009@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91546:e9b28081c7e0
Date: 2017-06-06 00:03 +0300
http://bitbucket.org/pypy/pypy/changeset/e9b28081c7e0/

Log:	tweak release notice

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,8 +10,9 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-This release enables link-time optimization and `profile guided optimization` 
-of the base interpreter, which may make unjitted code run faster.
+This release enables link-time optimization and `profile guided optimization`_
+of the base interpreter, which may make unjitted code run faster. To use these,
+translate with appropriate `options`_.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
@@ -49,6 +50,7 @@
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: project-ideas.html
+.. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
 
 What is PyPy?
@@ -148,7 +150,7 @@
     accepted in a few more places, e.g. in compile()
 
 
-.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+.. _here: cpython_differences.html
 
 Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
 ======================================================================

From pypy.commits at gmail.com  Tue Jun  6 01:47:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 22:47:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: Say clearly that LTO and PGO are
 disabled by default. Point to an issue
Message-ID: <593641fe.3098df0a.f3c63.b92f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91547:17a5b2dd6cec
Date: 2017-06-06 07:47 +0200
http://bitbucket.org/pypy/pypy/changeset/17a5b2dd6cec/

Log:	Say clearly that LTO and PGO are disabled by default. Point to an
	issue I created to focus discussion

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,9 +10,10 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-This release enables link-time optimization and `profile guided optimization`_
-of the base interpreter, which may make unjitted code run faster. To use these,
-translate with appropriate `options`_.
+This release adds (but disables by default) link-time optimization and
+`profile guided optimization`_ of the base interpreter, which may make
+unjitted code run faster. To use these, translate with appropriate
+`options`_.  Be aware of `[1]`_, though.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
@@ -44,6 +45,7 @@
 with making RPython's JIT even better.
 
 .. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
+.. _`[1]`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html

From pypy.commits at gmail.com  Tue Jun  6 01:51:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 05 Jun 2017 22:51:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: v2.7 => 2.7, for consistency
Message-ID: <593642fa.86941c0a.fb72.01b8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91548:46ae38723852
Date: 2017-06-06 07:51 +0200
http://bitbucket.org/pypy/pypy/changeset/46ae38723852/

Log:	v2.7 => 2.7, for consistency

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -3,8 +3,8 @@
 =====================================
 
 The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
-Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
-v3.5 syntax). The two releases are both based on much the same codebase, thus
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and

From pypy.commits at gmail.com  Tue Jun  6 08:13:15 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Tue, 06 Jun 2017 05:13:15 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: merge default
Message-ID: <59369c5b.0a5e1c0a.79ea8.5cb8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91549:913967a77f88
Date: 2017-06-06 08:12 -0400
http://bitbucket.org/pypy/pypy/changeset/913967a77f88/

Log:	merge default

diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -101,28 +101,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -149,13 +149,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -164,11 +164,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -183,6 +183,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -220,12 +221,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -264,7 +267,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -303,6 +305,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -340,11 +343,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.4'
+version = '5.8'
 # The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -68,28 +68,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -116,13 +116,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -131,11 +131,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -150,6 +150,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -187,12 +188,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -231,7 +234,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -270,6 +272,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -307,11 +310,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -60,7 +60,7 @@
 The interface for full finalizers is made with PyPy in mind, but should
 be generally useful.
 
-The idea is that you subclass the ``rgc.FinalizerQueue`` class::
+The idea is that you subclass the ``rgc.FinalizerQueue`` class:
 
 * You must give a class-level attribute ``base_class``, which is the
   base class of all instances with a finalizer.  (If you need
diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst
--- a/pypy/doc/discussion/rawrefcount.rst
+++ b/pypy/doc/discussion/rawrefcount.rst
@@ -68,10 +68,12 @@
 and O = list of links created with rawrefcount.create_link_pyobj().
 The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all
 the data is in the PyObjects, and all outsite references (if any) are
-in C, as "PyObject *" fields.
+in C, as ``PyObject *`` fields.
 
 So, during the collection we do this about P links:
 
+.. code-block:: python
+
     for (p, ob) in P:
         if ob->ob_refcnt != REFCNT_FROM_PYPY
                and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT:
@@ -80,6 +82,8 @@
 At the end of the collection, the P and O links are both handled like
 this:
 
+.. code-block:: python
+
     for (p, ob) in P + O:
         if p is not surviving:    # even if 'ob' might be surviving
             unlink p and ob
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -5,8 +5,8 @@
 ++++++++++++++
 
 We try to create a stable release a few times a year. These are released on
-a branch named like release-2.x or release-4.x, and each release is tagged,
-for instance release-4.0.1. 
+a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
+release is tagged, for instance release-pypy3.5-v4.0.1. 
 
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
@@ -33,7 +33,7 @@
 * If needed, make a release branch
 * Bump the
   pypy version number in module/sys/version.py and in
-  module/cpyext/include/patchlevel.h and . The branch
+  module/cpyext/include/patchlevel.h and in doc/conf.py. The branch
   will capture the revision number of this change for the release.
 
   Some of the next updates may be done before or after branching; make
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -30,12 +30,22 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+CPython 3.5 compatible versions
+-------------------------------
+
+.. toctree::
+
+   whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.8.0.rst
+   whatsnew-pypy3-5.7.0.rst
+
 CPython 3.3 compatible versions
 -------------------------------
 
 .. toctree::
 
    whatsnew-pypy3-5.5.0.rst
+   whatsnew-pypy3-5.1.1-alpha1.rst
 
 CPython 3.2 compatible versions
 -------------------------------
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -12,6 +12,7 @@
 and using pip. 
 
 .. _prebuilt-pypy:
+
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -250,12 +250,12 @@
 .. py:function:: newunicode(ustr)
 
    Creates a Unicode string from an rpython unicode string.
-   This method may disappear soon and be replaced by :py:function:`newutf8()`.
+   This method may disappear soon and be replaced by :py:function::`newutf8`.
 
 .. py:function:: newutf8(bytestr)
 
    Creates a Unicode string from an rpython byte string, decoded as
-   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function::`newtext`.
 
 Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
 `pypy/objspace/std/objspace.py`.
@@ -302,9 +302,9 @@
 
 .. py:function:: unicode_w(w_x)
 
-   Takes an application level :py:class:`unicode` and return an
+   Takes an application level :py:class::`unicode` and return an
    interpreter-level unicode string.  This method may disappear soon and
-   be replaced by :py:function:`text_w()`.
+   be replaced by :py:function::`text_w`.
 
 .. py:function:: float_w(w_x)
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -171,7 +171,7 @@
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
-   `malloc_nursery`, to move values from registers into other registers
+    `malloc_nursery`, to move values from registers into other registers
     instead of to the stack.
 
   * More tightly pack the stack when calling with `release gil`
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -140,7 +140,7 @@
     preamble
   * In JIT residual calls, if the called function starts with a fast-path like
     ``if x.foo != 0: return x.foo``, then inline the check before doing the
-     ``CALL``.
+    ``CALL``.
   * Ensure ``make_inputargs`` fails properly when given arguments with type 
     information
   * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -3,15 +3,17 @@
 =====================================
 
 The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
-Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
-v3.5 syntax). The two releases are both based on much the same codebase, thus
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-This release enables `profile guided optimization` of the base interpreter,
-which may make unjitted code run faster.
+This release adds (but disables by default) link-time optimization and
+`profile guided optimization`_ of the base interpreter, which may make
+unjitted code run faster. To use these, translate with appropriate
+`options`_.  Be aware of `[1]`_, though.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
@@ -43,12 +45,14 @@
 with making RPython's JIT even better.
 
 .. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
+.. _`[1]`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: project-ideas.html
+.. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
 
 What is PyPy?
@@ -148,7 +152,7 @@
     accepted in a few more places, e.g. in compile()
 
 
-.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+.. _here: cpython_differences.html
 
 Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
 ======================================================================
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Richard Plangger':['planrich'],
     'Michael Hudson': ['mwh'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
     "Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
-    'Stanislaw Halik':['w31rd0'],
+    'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
     'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
     'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
     'Jasper Schulz':['Jasper.Schulz', 'jbs'],
     'Aaron Gallagher':['"Aaron Gallagher'],
     'Yasir Suhail':['yasirs'],
-    'Squeaky', ['squeaky'],
+    'Squeaky': ['squeaky'],
+    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -6,6 +6,7 @@
 .. startrev: 91904d5c5188
 
 .. branch: use_min_scalar
+
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
 
 .. branch: stdlib-2.7.10
@@ -15,6 +16,7 @@
 .. branch: issue2062
 
 .. branch: disable-unroll-for-short-loops
+
 The JIT no longer performs loop unrolling if the loop compiles to too much code.
 
 .. branch: run-create_cffi_imports
@@ -32,9 +34,11 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+
 Part of cffi 1.2.
 
 .. branch: cffi-new-allocator
+
 Part of cffi 1.2.
 
 .. branch: unicode-dtype
diff --git a/pypy/doc/whatsnew-4.0.0.rst b/pypy/doc/whatsnew-4.0.0.rst
--- a/pypy/doc/whatsnew-4.0.0.rst
+++ b/pypy/doc/whatsnew-4.0.0.rst
@@ -6,23 +6,28 @@
 .. startrev: 3a8f5481dab4
 
 .. branch: keys_with_hash
+
 Improve the performance of ``dict.update()`` and a bunch of methods from
 sets, by reusing the hash value stored in one dict when inspecting
 or changing another dict with that key.
 
 .. branch: optresult-unroll 
+
 A major refactoring of the ``ResOperations`` that kills Box. Also rewrote
 unrolling to enable future enhancements.  Should improve warmup time
 by 20% or so.
 
 .. branch: optimize-cond-call
+
 Optimize common sequences of operations like
 ``int_lt/cond_call`` in the JIT backends
 
 .. branch: missing_openssl_include
+
 Fix for missing headers in OpenBSD, already applied in downstream ports
 
 .. branch: gc-more-incremental
+
 Remove a source of non-incremental-ness in the GC: now
 ``external_malloc()`` no longer runs ``gc_step_until()`` any more. If there
 is a currently-running major collection, we do only so many steps
@@ -32,11 +37,13 @@
 keep adding up between them.
 
 .. branch: remember-tracing-counts
+
 Reenable jithooks
 
 .. branch: detect_egd2
 
 .. branch: shadowstack-no-move-2
+
 Issue #2141: fix a crash on Windows and OS/X and ARM when running
 at least 20 threads.
 
@@ -55,6 +62,7 @@
 floats, cf. issue #2148.
 
 .. branch: cffi-stdcall
+
 Win32: support ``__stdcall`` in CFFI.
 
 .. branch: callfamily
@@ -93,6 +101,7 @@
 .. branch: osx-libffi
 
 .. branch: lazy-fast2locals
-improve the performance of simple trace functions by lazily calling
+
+Improve the performance of simple trace functions by lazily calling
 ``fast2locals`` and ``locals2fast`` only if ``f_locals`` is actually accessed.
 
diff --git a/pypy/doc/whatsnew-5.0.0.rst b/pypy/doc/whatsnew-5.0.0.rst
--- a/pypy/doc/whatsnew-5.0.0.rst
+++ b/pypy/doc/whatsnew-5.0.0.rst
@@ -192,6 +192,7 @@
 Fix boolean-array indexing in micronumpy
 
 .. branch: numpy_partition
+
 Support ndarray.partition() as an app-level function numpy.core._partition_use,
 provided as a cffi wrapper to upstream's implementation in the pypy/numpy repo
 
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -29,6 +29,7 @@
 upstream numpy via cpyext, so we created (yet another) fork of numpy at 
 github.com/pypy/numpy with the needed changes. Among the significant changes 
 to cpyext:
+
   - allow c-snippet tests to be run with -A so we can verify we are compatible
   - fix many edge cases exposed by fixing tests to run with -A
   - issequence() logic matches cpython
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -6,10 +6,12 @@
 .. startrev: 873218a739f1
 
 .. 418b05f95db5
+
 Improve CPython compatibility for ``is``. Now code like ``if x is ():``
 works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
 .. pull request #455
+
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 
 .. branch: fix-gen-dfa
@@ -36,9 +38,11 @@
 compatible.
 
 .. branch: pyfile-tell
+
 Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
 
 .. branch: rw-PyString_AS_STRING
+
 Allow rw access to the char* returned from PyString_AS_STRING, also refactor
 PyStringObject to look like cpython's and allow subclassing PyString_Type and
 PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.6.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -6,18 +6,22 @@
 .. startrev: 522736f816dc
 
 .. branch: rpython-resync
+
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
 .. branch: buffer-interface
+
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
 .. branch: force-virtual-state
+
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
 
 .. branch: conditional_call_value_3
+
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
@@ -58,6 +62,7 @@
 
 
 .. fb6bb835369e
+
 Change the ``timeit`` module: it now prints the average time and the standard
 deviation over 7 runs by default, instead of the minimum. The minimum is often
 misleading.
@@ -69,9 +74,6 @@
 
 .. branch: Tiberiumk/fix-2412-1476011166874
 .. branch: redirect-assembler-jitlog
-
-
-
 .. branch: stdlib-2.7.12
 
 Update stdlib to version 2.7.12
diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
--- a/pypy/doc/whatsnew-pypy3-5.8.0.rst
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -6,6 +6,7 @@
 .. startrev: afbf09453369
 
 .. branch: mtest
+
 Use "<python> -m test" to run the CPython test suite, as documented by CPython,
 instead of our outdated regrverbose.py script.
 

From pypy.commits at gmail.com  Tue Jun  6 12:13:53 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 06 Jun 2017 09:13:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <5936d4c1.02512e0a.6b85b.d597@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91550:b30914a448be
Date: 2017-06-06 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/b30914a448be/

Log:	fix test

diff --git a/pypy/module/imp/test/test_import.py b/pypy/module/imp/test/test_import.py
--- a/pypy/module/imp/test/test_import.py
+++ b/pypy/module/imp/test/test_import.py
@@ -781,11 +781,10 @@
     def test_abi_tag(self):
         space1 = maketestobjspace(make_config(None, soabi='TEST'))
         space2 = maketestobjspace(make_config(None, soabi=''))
+        assert importing.get_so_extension(space1).startswith('.TEST')
         if sys.platform == 'win32':
-            assert importing.get_so_extension(space1) == '.TESTi.pyd'
             assert importing.get_so_extension(space2) == '.pyd'
         else:
-            assert importing.get_so_extension(space1) == '.TESTi.so'
             assert importing.get_so_extension(space2) == '.so'
 
 def _getlong(data):

From pypy.commits at gmail.com  Wed Jun  7 02:06:00 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Tue, 06 Jun 2017 23:06:00 -0700 (PDT)
Subject: [pypy-commit] pypy ctypes_char_indexing: no string if we're array
 of char
Message-ID: <593797c8.4f572e0a.45a0c.09ed@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: ctypes_char_indexing
Changeset: r91551:dd4db811a46e
Date: 2017-05-29 16:25 +0300
http://bitbucket.org/pypy/pypy/changeset/dd4db811a46e/

Log:	no string if we're array of char

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -76,12 +76,6 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
-        # this seems to be a string if we're array of char, surprise!
-        from ctypes import c_char, c_wchar
-        if self._type_ is c_char:
-            return _rawffi.charp2string(resarray.buffer, self._length_)
-        if self._type_ is c_wchar:
-            return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray

From pypy.commits at gmail.com  Wed Jun  7 02:06:07 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Tue, 06 Jun 2017 23:06:07 -0700 (PDT)
Subject: [pypy-commit] pypy ctypes_char_indexing: indentation fix
Message-ID: <593797cf.c6aa190a.bd7d3.0a39@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: ctypes_char_indexing
Changeset: r91554:2ea696d9721d
Date: 2017-05-30 10:18 +0300
http://bitbucket.org/pypy/pypy/changeset/2ea696d9721d/

Log:	indentation fix

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -80,12 +80,12 @@
         # If a char_p or unichar_p is received, skip the string interpretation
         if base._ffiargtype != types.Pointer(types.char_p) and \
            base._ffiargtype != types.Pointer(types.unichar_p):
-            #this seems to be a string if we're array of char, surprise!
+            # this seems to be a string if we're array of char, surprise!
             from ctypes import c_char, c_wchar
             if self._type_ is c_char:
-               return _rawffi.charp2string(resarray.buffer, self._length_)
+                return _rawffi.charp2string(resarray.buffer, self._length_)
             if self._type_ is c_wchar:
-               return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+                return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray

From pypy.commits at gmail.com  Wed Jun  7 02:06:02 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Tue, 06 Jun 2017 23:06:02 -0700 (PDT)
Subject: [pypy-commit] pypy ctypes_char_indexing: fix char_p and unichar_p
 interpretation
Message-ID: <593797ca.511d190a.722b1.0cb3@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: ctypes_char_indexing
Changeset: r91552:c5adea5c7b06
Date: 2017-05-29 21:11 +0300
http://bitbucket.org/pypy/pypy/changeset/c5adea5c7b06/

Log:	fix char_p and unichar_p interpretation

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -76,6 +76,16 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
+        from _rawffi.alt import types
+        # If a char_p or unichar_p is received, skip the string interpretation
+        if base._ffiargtype != types.Pointer(types.char_p) and \
+           base._ffiargtype != types.Pointer(types.unichar_p):
+            #this seems to be a string if we're array of char, surprise!
+            from ctypes import c_char, c_wchar
+            if self._type_ is c_char:
+               return _rawffi.charp2string(resarray.buffer, self._length_)
+            if self._type_ is c_wchar:
+               return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray

From pypy.commits at gmail.com  Wed Jun  7 02:06:09 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Jun 2017 23:06:09 -0700 (PDT)
Subject: [pypy-commit] pypy default: Merged in
 palecsandru/pypy_ctypes_char_indexing/ctypes_char_indexing (pull request
 #552)
Message-ID: <593797d1.58212e0a.5b8f.0783@mx.google.com>

Author: Armin Rigo <armin.rigo at gmail.com>
Branch: 
Changeset: r91555:60d070027d70
Date: 2017-06-07 06:05 +0000
http://bitbucket.org/pypy/pypy/changeset/60d070027d70/

Log:	Merged in palecsandru/pypy_ctypes_char_indexing/ctypes_char_indexing
	(pull request #552)

	Indexing into char* behaves differently than CPython

diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -76,12 +76,16 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
-        # this seems to be a string if we're array of char, surprise!
-        from ctypes import c_char, c_wchar
-        if self._type_ is c_char:
-            return _rawffi.charp2string(resarray.buffer, self._length_)
-        if self._type_ is c_wchar:
-            return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+        from _rawffi.alt import types
+        # If a char_p or unichar_p is received, skip the string interpretation
+        if base._ffiargtype != types.Pointer(types.char_p) and \
+           base._ffiargtype != types.Pointer(types.unichar_p):
+            # this seems to be a string if we're array of char, surprise!
+            from ctypes import c_char, c_wchar
+            if self._type_ is c_char:
+                return _rawffi.charp2string(resarray.buffer, self._length_)
+            if self._type_ is c_wchar:
+                return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
@@ -138,4 +138,39 @@
         x.y = 3
         y[1] = x
         assert y[1].y == 3
+
+    def test_output_simple(self):
+        A = c_char * 10
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+        A = c_wchar * 10
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+    def test_output_complex_test(self):
+        class Car(Structure):
+            _fields_ = [("brand", c_char * 10),
+                        ("speed", c_float),
+                        ("owner", c_char * 10)]
+
+        assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi"
+        assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio"
+        raises(ValueError, Car, "abcdefghiop", 42.0, "12345")
+
+        A = Car._fields_[2][1]
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+        c = Car()
+        c.brand = "abcdex"
+        c.speed = 42.0
+        c.owner = "12345"
+        assert isinstance(repr(c.brand), bytes)
         

From pypy.commits at gmail.com  Wed Jun  7 02:06:05 2017
From: pypy.commits at gmail.com (Alecsandru Patrascu)
Date: Tue, 06 Jun 2017 23:06:05 -0700 (PDT)
Subject: [pypy-commit] pypy ctypes_char_indexing: added tests for char_p and
 unichar_p interpretation
Message-ID: <593797cd.c3092e0a.dc6e8.0ad8@mx.google.com>

Author: Alecsandru Patrascu <alecsandru.patrascu at rinftech.com>
Branch: ctypes_char_indexing
Changeset: r91553:76bf604679cd
Date: 2017-05-30 09:58 +0300
http://bitbucket.org/pypy/pypy/changeset/76bf604679cd/

Log:	added tests for char_p and unichar_p interpretation

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
@@ -138,4 +138,39 @@
         x.y = 3
         y[1] = x
         assert y[1].y == 3
+
+    def test_output_simple(self):
+        A = c_char * 10
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+        A = c_wchar * 10
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+    def test_output_complex_test(self):
+        class Car(Structure):
+            _fields_ = [("brand", c_char * 10),
+                        ("speed", c_float),
+                        ("owner", c_char * 10)]
+
+        assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi"
+        assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio"
+        raises(ValueError, Car, "abcdefghiop", 42.0, "12345")
+
+        A = Car._fields_[2][1]
+        TP = POINTER(A)
+        x = TP(A())
+        assert repr(x[0]) != "''"
+        assert isinstance(repr(x[0]), bytes)
+
+        c = Car()
+        c.brand = "abcdex"
+        c.speed = 42.0
+        c.owner = "12345"
+        assert isinstance(repr(c.brand), bytes)
         

From pypy.commits at gmail.com  Wed Jun  7 02:16:24 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 06 Jun 2017 23:16:24 -0700 (PDT)
Subject: [pypy-commit] pypy default: Simplify and complete the tests
Message-ID: <59379a38.02452e0a.e0cad.0c8d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91556:e2b62fc32b15
Date: 2017-06-07 08:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e2b62fc32b15/

Log:	Simplify and complete the tests

diff --git a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
--- a/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
+++ b/pypy/module/test_lib_pypy/ctypes_tests/test_array.py
@@ -143,14 +143,18 @@
         A = c_char * 10
         TP = POINTER(A)
         x = TP(A())
-        assert repr(x[0]) != "''"
-        assert isinstance(repr(x[0]), bytes)
+        assert x[0] != ''
 
         A = c_wchar * 10
         TP = POINTER(A)
         x = TP(A())
-        assert repr(x[0]) != "''"
-        assert isinstance(repr(x[0]), bytes)
+        assert x[0] != ''
+
+    def test_output_simple_array(self):
+        A = c_char * 10
+        AA = A * 10
+        aa = AA()
+        assert aa[0] != ''
 
     def test_output_complex_test(self):
         class Car(Structure):
@@ -158,6 +162,7 @@
                         ("speed", c_float),
                         ("owner", c_char * 10)]
 
+        assert isinstance(Car("abcdefghi", 42.0, "12345").brand, bytes)
         assert Car("abcdefghi", 42.0, "12345").brand == "abcdefghi"
         assert Car("abcdefghio", 42.0, "12345").brand == "abcdefghio"
         raises(ValueError, Car, "abcdefghiop", 42.0, "12345")
@@ -165,12 +170,4 @@
         A = Car._fields_[2][1]
         TP = POINTER(A)
         x = TP(A())
-        assert repr(x[0]) != "''"
-        assert isinstance(repr(x[0]), bytes)
-
-        c = Car()
-        c.brand = "abcdex"
-        c.speed = 42.0
-        c.owner = "12345"
-        assert isinstance(repr(c.brand), bytes)
-        
+        assert x[0] != ''

From pypy.commits at gmail.com  Wed Jun  7 06:59:53 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Jun 2017 03:59:53 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: missing vmprof_ prefix
Message-ID: <5937dca9.cf10190a.cf9af.1d94@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91557:876d67108576
Date: 2017-06-07 06:58 -0400
http://bitbucket.org/pypy/pypy/changeset/876d67108576/

Log:	missing vmprof_ prefix

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -98,8 +98,8 @@
     return space.newtext(path)
 
 def stop_sampling(space):
-    return space.newint(rvmprof.stop_sampling(space))
+    return space.newint(rvmprof.vmprof_stop_sampling(space))
 
 def start_sampling(space):
-    rvmprof.start_sampling(space)
+    rvmprof.vmprof_start_sampling(space)
     return space.w_None

From pypy.commits at gmail.com  Wed Jun  7 07:06:00 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Jun 2017 04:06:00 -0700 (PDT)
Subject: [pypy-commit] pypy default: Tweak to also report the single largest
 objects
Message-ID: <5937de18.8cea190a.904b9.1f13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91558:7f9ffd5232c6
Date: 2017-06-07 13:05 +0200
http://bitbucket.org/pypy/pypy/changeset/7f9ffd5232c6/

Log:	Tweak to also report the single largest objects

diff --git a/pypy/tool/gcdump.py b/pypy/tool/gcdump.py
--- a/pypy/tool/gcdump.py
+++ b/pypy/tool/gcdump.py
@@ -13,10 +13,12 @@
 class Stat(object):
     summary = {}
     typeids = {0: '<GCROOT>'}
+    BIGOBJ = 65536   # bytes
 
     def summarize(self, filename):
         a = self.load_dump_file(filename)
         self.summary = {}     # {typenum: [count, totalsize]}
+        self.bigobjs = []     # list of individual (size, typenum)
         for obj in self.walk(a):
             self.add_object_summary(obj[2], obj[3])
 
@@ -50,6 +52,18 @@
             print '%8d %8.2fM  %s' % (stat[0], stat[1] / (1024.0*1024.0),
                                       self.get_type_name(typenum))
         print 'total %.1fM' % (totalsize / (1024.0*1024.0),)
+        print
+        lst = sorted(self.bigobjs)[-10:]
+        if lst:
+            if len(lst) == len(self.bigobjs):
+                print '%d objects take at least %d bytes each:' % (len(lst), self.BIGOBJ)
+            else:
+                print '%d largest single objects:' % (len(lst),)
+            for size, typenum in lst:
+                print '%8s %8.2fM  %s' % ('', size / (1024.0*1024.0),
+                                          self.get_type_name(typenum))
+        else:
+            print 'No object takes at least %d bytes on its own.' % (self.BIGOBJ,)
 
     def load_dump_file(self, filename):
         f = open(filename, 'rb')
@@ -62,6 +76,8 @@
         return a
 
     def add_object_summary(self, typenum, sizeobj):
+        if sizeobj >= self.BIGOBJ:
+            self.bigobjs.append((sizeobj, typenum))
         try:
             stat = self.summary[typenum]
         except KeyError:

From pypy.commits at gmail.com  Wed Jun  7 07:06:38 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Jun 2017 04:06:38 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: added prefix vmprof_ to wrong
 callsite
Message-ID: <5937de3e.41be190a.96ce1.1e68@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91559:ba7e4af9836a
Date: 2017-06-07 07:05 -0400
http://bitbucket.org/pypy/pypy/changeset/ba7e4af9836a/

Log:	added prefix vmprof_ to wrong callsite

diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -98,8 +98,8 @@
     return space.newtext(path)
 
 def stop_sampling(space):
-    return space.newint(rvmprof.vmprof_stop_sampling(space))
+    return space.newint(rvmprof.stop_sampling(space))
 
 def start_sampling(space):
-    rvmprof.vmprof_start_sampling(space)
+    rvmprof.start_sampling(space)
     return space.w_None
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -56,8 +56,8 @@
     return None
 
 def stop_sampling(space):
-    fd = _get_vmprof().cintf.stop_sampling()
+    fd = _get_vmprof().cintf.vmprof_stop_sampling()
     return rffi.cast(lltype.Signed, fd)
 
 def start_sampling(space):
-    _get_vmprof().cintf.start_sampling()
+    _get_vmprof().cintf.vmprof_start_sampling()

From pypy.commits at gmail.com  Wed Jun  7 10:15:14 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 07:15:14 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy2.7-v5.8.0 for
 changeset c925e7381036
Message-ID: <59380a72.18542e0a.1d9e2.2a5e@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91560:8e0f40a44beb
Date: 2017-06-07 16:19 +0300
http://bitbucket.org/pypy/pypy/changeset/8e0f40a44beb/

Log:	Added tag release-pypy2.7-v5.8.0 for changeset c925e7381036

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -38,3 +38,4 @@
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
+c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0

From pypy.commits at gmail.com  Wed Jun  7 10:15:17 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 07:15:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Added tag release-pypy3.5-v5.8.0 for
 changeset a37ecfe5f142
Message-ID: <59380a75.8348190a.b83b7.269f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91561:d17da6177617
Date: 2017-06-07 16:20 +0300
http://bitbucket.org/pypy/pypy/changeset/d17da6177617/

Log:	Added tag release-pypy3.5-v5.8.0 for changeset a37ecfe5f142

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -39,3 +39,4 @@
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
 c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
+a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0

From pypy.commits at gmail.com  Wed Jun  7 10:15:19 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 07:15:19 -0700 (PDT)
Subject: [pypy-commit] pypy default: repackage script - mroe robust and
 quieter, check hg hash, refactor for 2.7/3.5
Message-ID: <59380a77.02512e0a.36c5d.2a32@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91562:a0bfbbf2ecb2
Date: 2017-06-07 17:14 +0300
http://bitbucket.org/pypy/pypy/changeset/a0bfbbf2ecb2/

Log:	repackage script - mroe robust and quieter, check hg hash, refactor
	for 2.7/3.5

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -1,24 +1,44 @@
 # Edit these appropriately before running this script
+pmaj=2  # python main version
+pmin=7  # python minor version
 maj=5
-min=7
-rev=1
-branchname=release-pypy2.7-5.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
-tagname=release-pypy2.7-v$maj.$min.$rev  # ==OR== release-$maj.$min
+min=8
+rev=0
+branchname=release-pypy$pmaj.$pmin-$maj.x # ==OR== release-$maj.x  # ==OR== release-$maj.$min.x
+tagname=release-pypy$pmaj.$pmin-v$maj.$min.$rev  # ==OR== release-$maj.$min
 
 echo checking hg log -r $branchname
 hg log -r $branchname || exit 1
 echo checking hg log -r $tagname
 hg log -r $tagname || exit 1
+hgrev=`hg id -r $tagname -i`
 
-rel=pypy2-v$maj.$min.$rev
-# This script will download latest builds from the buildmaster, rename the top
-# level directory, and repackage ready to be uploaded to bitbucket. It will also
-# download source, assuming a tag for the release already exists, and repackage them.
-# The script should be run in an empty directory, i.e. /tmp/release_xxx
+rel=pypy$pmaj-v$maj.$min.$rev
+# The script should be run in an empty in the pypy tree, i.e. pypy/tmp
+if [ "`ls . | wc -l`" != "0" ]
+then
+    echo this script must be run in an empty directory
+    exit -1
+fi
+
+# Download latest builds from the buildmaster, rename the top
+# level directory, and repackage ready to be uploaded to bitbucket
 for plat in linux linux64 linux-armhf-raspbian linux-armhf-raring linux-armel osx64 s390x
   do
     echo downloading package for $plat
-    wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.tar.bz2
+    if wget -q --show-progress http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.tar.bz2
+    then
+        echo $plat downloaded 
+    else
+        echo $plat no download available
+        continue
+    fi
+    hgcheck=`tar -tf pypy-c-jit-latest-$plat.tar.bz2 |head -n1 | cut -d- -f5`
+    if [ "$hgcheck" != "$hgrev" ]
+    then
+        echo $plat hg tag mismatch, expected $hgrev, got $hgcheck
+        continue
+    fi
     tar -xf pypy-c-jit-latest-$plat.tar.bz2
     rm pypy-c-jit-latest-$plat.tar.bz2
     plat_final=$plat
@@ -32,23 +52,31 @@
   done
 
 plat=win32
-wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip
-unzip pypy-c-jit-latest-$plat.zip
-rm pypy-c-jit-latest-$plat.zip
-mv pypy-c-jit-*-$plat $rel-$plat
-zip -rq $rel-$plat.zip $rel-$plat
-rm -rf $rel-$plat
+if wget http://buildbot.pypy.org/nightly/$branchname/pypy-c-jit-latest-$plat.zip
+then
+    unzip -q pypy-c-jit-latest-$plat.zip
+    rm pypy-c-jit-latest-$plat.zip
+    mv pypy-c-jit-*-$plat $rel-$plat
+    zip -rq $rel-$plat.zip $rel-$plat
+    rm -rf $rel-$plat
+else
+    echo no download for $plat
+fi
 
+# Download source and repackage
 # Requires a valid $tagname, note the untarred directory is pypy-pypy-<hash>
 # so make sure there is not another one
-wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
-tar -xf $tagname.tar.bz2
-rm $tagname.tar.bz2
-mv pypy-pypy-* $rel-src
-tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src
-zip -rq $rel-src.zip $rel-src
-rm -rf $rel-src
-
+if wget https://bitbucket.org/pypy/pypy/get/$tagname.tar.bz2
+then
+    tar -xf $tagname.tar.bz2
+    rm $tagname.tar.bz2
+    mv pypy-pypy-* $rel-src
+    tar --owner=root --group=root --numeric-owner -cjf $rel-src.tar.bz2 $rel-src
+    zip -rq $rel-src.zip $rel-src
+    rm -rf $rel-src
+else
+    echo source tarfile for $tagname not found on bitbucket, did you push the tag commit?
+fi
 # Print out the md5, sha1, sha256
 #md5sum *.bz2 *.zip
 #sha1sum *.bz2 *.zip

From pypy.commits at gmail.com  Wed Jun  7 10:36:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 07:36:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: copy packaging explaination from
 www.pypy.org to documentation
Message-ID: <59380f54.54a1190a.3ae3d.2ce8@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91563:197b78d9963b
Date: 2017-06-07 17:35 +0300
http://bitbucket.org/pypy/pypy/changeset/197b78d9963b/

Log:	copy packaging explaination from www.pypy.org to documentation

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -195,6 +195,29 @@
 ``/tmp/usession-YOURNAME/build/``.  You can then either move the file
 hierarchy or unpack the ``.tar.bz2`` at the correct place.
 
+It is recommended to use package.py because custom scripts will
+invariably become out-of-date.  If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation.  If you install PyPy as root without
+pre-compiling them, normal users will get errors:
+
+* PyPy 2.5.1 or earlier: normal users would see permission errors.
+  Installers need to run ``pypy -c "import gdbm"`` and other similar
+  commands at install time; the exact list is in `package.py`_.  Users
+  seeing a broken installation of PyPy can fix it after-the-fact if they
+  have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
+
+* PyPy 2.6 and later: anyone would get ``ImportError: no module named
+  _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
+  ``lib_pypy`` directory during the installation process (plus others;
+  see the exact list in `package.py`_).  Users seeing a broken
+  installation of PyPy can fix it after-the-fact, by running ``pypy
+  /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
+  called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
+  module for PyPy.  You can move it at any place where modules are
+  normally found: e.g. in your project's main directory, or in a
+  directory that you add to the env var ``PYTHONPATH``.
+
 
 Installation
 ------------

From pypy.commits at gmail.com  Wed Jun  7 10:37:30 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 07:37:30 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update for 5.8.0,
 leave 5.7 hashes for now
Message-ID: <59380faa.11d0190a.4c06c.2b12@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r886:344a346894fb
Date: 2017-06-07 17:36 +0300
http://bitbucket.org/pypy/pypy.org/changeset/344a346894fb/

Log:	update for 5.8.0, leave 5.7 hashes for now

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -15,14 +15,14 @@
 We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
 Linux, Mac OS X and Windows:
 
-* the Python2.7 compatible release &#8212; **PyPy2.7 v5.7.1** &#8212; (`what's new in PyPy2.7?`_)
+* the Python2.7 compatible release &#8212; **PyPy2.7 v5.8.0** &#8212; (`what's new in PyPy2.7?`_)
 
-* the Python3.5 compatible beta quality release &#8212; **PyPy3.5 v5.7.1** &#8212; (`what's new in PyPy3.5?`_).
+* the Python3.5 compatible beta quality release &#8212; **PyPy3.5 v5.8.0** &#8212; (`what's new in PyPy3.5?`_).
 
 * the Python2.7 Software Transactional Memory special release &#8212; **PyPy-STM 2.5.1** (Linux x86-64 only)
 
-.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v5.7.1.html
-.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.7.1.html
+.. _what's new in PyPy2.7?: http://doc.pypy.org/en/latest/release-v5.8.0.html
+.. _what's new in PyPy3.5?: http://doc.pypy.org/en/latest/release-v5.8.0.html
 
 
 .. class:: download_menu
@@ -79,7 +79,7 @@
 
 .. _release:
 
-Python2.7 compatible PyPy 5.7.1
+Python2.7 compatible PyPy 5.8.0
 -------------------------------
 
 * `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
@@ -98,24 +98,24 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux32.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raspbian.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raring.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armel.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-osx64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-win32.zip
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-ppc64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-ppc64le.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-s390x.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux32.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armhf-raspbian.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armhf-raring.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armel.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-osx64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-win32.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-ppc64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-ppc64le.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-s390x.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.zip
 .. _`vcredist_x86.exe`: http://www.microsoft.com/en-us/download/details.aspx?id=5582
 .. __: https://bitbucket.org/pypy/pypy/downloads
 .. _mirror: http://buildbot.pypy.org/mirror/
 .. _FreshPorts: http://www.freshports.org/lang/pypy
 
-Python 3.5.3 compatible PyPy3.5 v5.7
+Python 3.5.3 compatible PyPy3.5 v5.8
 ------------------------------------
 
 .. class:: download_menu
@@ -131,9 +131,9 @@
 * `All our downloads,`__ including previous versions.  We also have a
   mirror_, but please use only if you have troubles accessing the links above
 
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-src.tar.bz2
-.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-src.zip
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-linux64.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-src.tar.bz2
+.. __: https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-src.zip
 .. __: https://bitbucket.org/pypy/pypy/downloads
 
 If your CPU is really, really old, it may be a x86-32 without SSE2.
@@ -203,7 +203,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in ``/opt``, and
 if you want, put a symlink from somewhere like
-``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.7.1/bin/pypy``.  Do
+``/usr/local/bin/pypy`` to ``/path/to/pypy2-5.8.0/bin/pypy``.  Do
 not move or copy the executable ``pypy`` outside the tree --- put
 a symlink to it, otherwise it will not find its libraries.
 
@@ -307,9 +307,9 @@
    Alternatively, the following smaller package contains the source at
    the same revision as the above binaries:
 
-   * `pypy2-v5.7.1-src.tar.bz2`__ (sources)
+   * `pypy2-v5.8.0-src.tar.bz2`__ (sources)
 
-   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.tar.bz2
+   .. __: https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.tar.bz2
 
 
 2. Make sure you **installed the dependencies.**  See the list here__.
@@ -439,6 +439,19 @@
 
 Here are the checksums for each of the downloads
 
+pypy2.7-v5.8.0 sha256::
+
+    a0b125a5781f7e5ddfc3baca46503b14f4ee6a0e234e8d72bfcf3afdf4120bef  pypy2-v5.8.0-linux32.tar.bz2
+    6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e  pypy2-v5.8.0-linux64.tar.bz2
+    28b7fd0cc7418ffc66c71520728e87941be40ebf4b82675c57e25598a2a702b0  pypy2-v5.8.0-linux-armel.tar.bz2
+    ddceca9c5c9a456d4bf1beab177660adffbbdf255a922244e1cc05f20318be46  pypy2-v5.8.0-linux-armhf-raring.tar.bz2
+    da58279a0e3706889fc0df06087cea08f8cfd22322139fe9bae73ef9b2d119b7  pypy2-v5.8.0-linux-armhf-raspbian.tar.bz2
+    04b61d1cf13aaca6d0420e854c820b8bd049dc88be16c02542abe8ca26eb075c  pypy2-v5.8.0-osx64.tar.bz2
+    35aea25e2b9d2f7c8742c47e4e7474ef0f93ce1b5e3d4f5a99795bab23c1ad2c  pypy2-v5.8.0-s390x.tar.bz2
+    504c2d522595baf8775ae1045a217a2b120732537861d31b889d47c340b58bd5  pypy2-v5.8.0-src.tar.bz2
+    ec1e34cc81a7f4086135bab29dcbe61d19fcd8d9d8fc1b149bea8373f94fd958  pypy2-v5.8.0-src.zip
+    43d6217653e5bdc09e3ff8cb56fb52c4eb019429063d80107be4e88eef79ea8d  pypy2-v5.8.0-win32.zip
+
 pypy2.7-v5.7.1 sha256::
 
     f125a227f8c814ba1698168a639ea6ca59bb69c280529639eed29076d8429a73  pypy2-v5.7.1-linux32.tar.bz2

From pypy.commits at gmail.com  Wed Jun  7 10:38:57 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Jun 2017 07:38:57 -0700 (PDT)
Subject: [pypy-commit] pypy default: merge branch vmprof-0.4.8
Message-ID: <59381001.85981c0a.ca055.e5b8@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: 
Changeset: r91564:1c08349b2ad3
Date: 2017-06-07 10:38 -0400
http://bitbucket.org/pypy/pypy/changeset/1c08349b2ad3/

Log:	merge branch vmprof-0.4.8

diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -14,6 +14,9 @@
         'write_all_code_objects': 'interp_vmprof.write_all_code_objects',
         'is_enabled': 'interp_vmprof.is_enabled',
         'get_profile_path': 'interp_vmprof.get_profile_path',
+        'stop_sampling': 'interp_vmprof.stop_sampling',
+        'start_sampling': 'interp_vmprof.start_sampling',
+
         'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
     }
 
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -96,3 +96,10 @@
         # Indicates an error! Assume platform does not implement the function call
         raise oefmt(space.w_NotImplementedError, "platform not implemented")
     return space.newtext(path)
+
+def stop_sampling(space):
+    return space.newint(rvmprof.stop_sampling(space))
+
+def start_sampling(space):
+    rvmprof.start_sampling(space)
+    return space.w_None
diff --git a/rpython/rlib/rvmprof/__init__.py b/rpython/rlib/rvmprof/__init__.py
--- a/rpython/rlib/rvmprof/__init__.py
+++ b/rpython/rlib/rvmprof/__init__.py
@@ -3,13 +3,12 @@
 from rpython.rlib.rvmprof.rvmprof import vmprof_execute_code, MAX_FUNC_NAME
 from rpython.rlib.rvmprof.rvmprof import _was_registered
 from rpython.rlib.rvmprof.cintf import VMProfPlatformUnsupported
-from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 #
 # See README.txt.
 #
 
-
 #vmprof_execute_code(): implemented directly in rvmprof.py
 
 def register_code_object_class(CodeClass, full_name_func):
@@ -56,3 +55,9 @@
 
     return None
 
+def stop_sampling(space):
+    fd = _get_vmprof().cintf.vmprof_stop_sampling()
+    return rffi.cast(lltype.Signed, fd)
+
+def start_sampling(space):
+    _get_vmprof().cintf.vmprof_start_sampling()
diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py
--- a/rpython/rlib/rvmprof/cintf.py
+++ b/rpython/rlib/rvmprof/cintf.py
@@ -110,6 +110,12 @@
     vmprof_get_profile_path = rffi.llexternal("vmprof_get_profile_path", [rffi.CCHARP, lltype.Signed],
                                               lltype.Signed, compilation_info=eci,
                                               _nowrapper=True)
+    vmprof_stop_sampling = rffi.llexternal("vmprof_stop_sampling", [],
+                                              rffi.INT, compilation_info=eci,
+                                              _nowrapper=True)
+    vmprof_start_sampling = rffi.llexternal("vmprof_start_sampling", [],
+                                              lltype.Void, compilation_info=eci,
+                                              _nowrapper=True)
 
     return CInterface(locals())
 
diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c
--- a/rpython/rlib/rvmprof/src/rvmprof.c
+++ b/rpython/rlib/rvmprof/src/rvmprof.c
@@ -30,8 +30,18 @@
 }
 #endif
 
-
 long vmprof_get_profile_path(const char * buffer, long size)
 {
     return vmp_fd_to_path(vmp_profile_fileno(), buffer, size);
 }
+
+int vmprof_stop_sampling(void)
+{
+    vmprof_ignore_signals(1);
+    return vmp_profile_fileno();
+}
+
+void vmprof_start_sampling(void)
+{
+    vmprof_ignore_signals(0);
+}
diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h
--- a/rpython/rlib/rvmprof/src/rvmprof.h
+++ b/rpython/rlib/rvmprof/src/rvmprof.h
@@ -38,6 +38,8 @@
 RPY_EXTERN void vmprof_stack_free(void*);
 RPY_EXTERN intptr_t vmprof_get_traceback(void *, void *, intptr_t*, intptr_t);
 RPY_EXTERN long vmprof_get_profile_path(const char *, long);
+RPY_EXTERN int vmprof_stop_sampling(void);
+RPY_EXTERN void vmprof_start_sampling(void);
 
 long vmprof_write_header_for_jit_addr(intptr_t *result, long n,
                                       intptr_t addr, int max_depth);
diff --git a/rpython/rlib/rvmprof/src/shared/_vmprof.c b/rpython/rlib/rvmprof/src/shared/_vmprof.c
--- a/rpython/rlib/rvmprof/src/shared/_vmprof.c
+++ b/rpython/rlib/rvmprof/src/shared/_vmprof.c
@@ -13,7 +13,6 @@
 static volatile int is_enabled = 0;
 static destructor Original_code_dealloc = 0;
 static PyObject* (*_default_eval_loop)(PyFrameObject *, int) = 0;
-void vmp_scan_profile(int fileno, int dump_native, void *all_code_uids);
 
 #if VMPROF_UNIX
 #include "trampoline.h"
@@ -82,15 +81,23 @@
     return vmprof_register_virtual_function(buf, CODE_ADDR_TO_UID(co), 500000);
 }
 
-static int _look_for_code_object(PyObject *o, void *all_codes)
+static int _look_for_code_object(PyObject *o, void * param)
 {
-    if (PyCode_Check(o) && !PySet_Contains((PyObject *)all_codes, o)) {
-        Py_ssize_t i;
+    int i;
+    PyObject * all_codes, * seen_codes;
+
+    all_codes = (PyObject*)((void**)param)[0];
+    seen_codes = (PyObject*)((void**)param)[1];
+    if (PyCode_Check(o) && !PySet_Contains(all_codes, o)) {
         PyCodeObject *co = (PyCodeObject *)o;
-        if (emit_code_object(co) < 0)
-            return -1;
-        if (PySet_Add((PyObject *)all_codes, o) < 0)
-            return -1;
+        PyObject * id = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co));
+        if (PySet_Contains(seen_codes, id)) {
+            // only emit if the code id has been seen!
+            if (emit_code_object(co) < 0)
+                return -1;
+            if (PySet_Add(all_codes, o) < 0)
+                return -1;
+        }
 
         /* as a special case, recursively look for and add code
            objects found in the co_consts.  The problem is that code
@@ -101,44 +108,19 @@
         while (i > 0) {
             --i;
             if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
-                                      all_codes) < 0)
+                                      param) < 0)
                 return -1;
         }
     }
     return 0;
 }
 
-static int _look_for_code_object_seen(PyObject *o, void *all_codes)
-{
-    if (PyCode_Check(o) && PySet_GET_SIZE(all_codes)) {
-        Py_ssize_t i;
-        PyCodeObject *co = (PyCodeObject *)o;
-        PyObject *uid_co = PyLong_FromVoidPtr((void*)CODE_ADDR_TO_UID(co));
-        int check = PySet_Discard(all_codes, uid_co);
-
-        Py_CLEAR(uid_co);
-
-        if (check < 0)
-            return -1;
-
-        if (check && emit_code_object(co) < 0)
-            return -1;
-
-        i = PyTuple_Size(co->co_consts);
-        while (i > 0) {
-            --i;
-            if (_look_for_code_object(PyTuple_GET_ITEM(co->co_consts, i),
-                                      all_codes) < 0)
-                return -1;
-        }
-    }
-    return 0;
-}
-
-static void emit_all_code_objects(void)
+static
+void emit_all_code_objects(PyObject * seen_code_ids)
 {
     PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
     Py_ssize_t i, size;
+    void * param[2];
 
     gc_module = PyImport_ImportModuleNoBlock("gc");
     if (gc_module == NULL)
@@ -154,11 +136,14 @@
     if (all_codes == NULL)
         goto error;
 
+    param[0] = all_codes;
+    param[1] = seen_code_ids;
+
     size = PyList_GET_SIZE(lst);
     for (i = 0; i < size; i++) {
         PyObject *o = PyList_GET_ITEM(lst, i);
         if (o->ob_type->tp_traverse &&
-            o->ob_type->tp_traverse(o, _look_for_code_object, (void *)all_codes)
+            o->ob_type->tp_traverse(o, _look_for_code_object, (void*)param)
                 < 0)
             goto error;
     }
@@ -169,91 +154,6 @@
     Py_XDECREF(gc_module);
 }
 
-static int add_code_addr(void *all_code_uids, void *addr)
-{
-    PyObject *co_uid = PyLong_FromVoidPtr(addr);
-    int check = PySet_Add((PyObject*) all_code_uids, co_uid);
-    Py_CLEAR(co_uid);
-    return check;
-}
-
-static void emit_all_code_objects_seen(int fileno)
-{
-    PyObject *gc_module = NULL, *lst = NULL, *all_codes = NULL;
-    Py_ssize_t i, size;
-
-    gc_module = PyImport_ImportModuleNoBlock("gc");
-    if (gc_module == NULL)
-        goto error;
-
-    lst = PyObject_CallMethod(gc_module, "get_objects", "");
-    if (lst == NULL || !PyList_Check(lst))
-        goto error;
-
-    all_codes = PySet_New(NULL);
-    if (all_codes == NULL)
-        goto error;
-
-    // fill up all_codes with every code object found in the profile
-    vmp_scan_profile(fileno, 0, all_codes);
-
-    // intersect the list with the set and dump only the code objects
-    // found in the set!
-    size = PyList_GET_SIZE(lst);
-    for (i = 0; i < size; i++) {
-        PyObject *o = PyList_GET_ITEM(lst, i);
-        if (o->ob_type->tp_traverse &&
-                o->ob_type->tp_traverse(o, _look_for_code_object_seen, (void *) all_codes)
-            < 0)
-            goto error;
-    }
-
-    error:
-    Py_XDECREF(all_codes);
-    Py_XDECREF(lst);
-    Py_XDECREF(gc_module);
-}
-
-static int emit_all_code_objects_helper(int only_needed, int disable_vmprof)
-{
-    int fd = vmp_profile_fileno();
-
-    if (!is_enabled) {
-        PyErr_SetString(PyExc_ValueError, "vmprof is not enabled");
-        return -1;
-    }
-
-#if VMPROF_UNIX
-    if ((read(fd, NULL, 0) != 0) && (only_needed != 0)) {
-        PyErr_SetString(PyExc_ValueError,
-                        "file descriptor must be readable to save only needed code objects");
-        return -1;
-    }
-#else
-    if (only_needed) {
-        PyErr_SetString(PyExc_ValueError,
-                        "saving only needed code objects is not supported for windows");
-        return -1;
-    }
-#endif
-
-    if (disable_vmprof) {
-        is_enabled = 0;
-        vmprof_ignore_signals(1);
-    }
-
-#if VMPROF_UNIX
-    if (only_needed)
-        emit_all_code_objects_seen(fd);
-    else
-        emit_all_code_objects();
-#else
-    emit_all_code_objects();
-#endif
-
-    return 0;
-}
-
 static void cpyprof_code_dealloc(PyObject *co)
 {
     if (is_enabled) {
@@ -283,7 +183,7 @@
     }
 
     if ((read(fd, NULL, 0) != 0) && (native != 0)) {
-        PyErr_SetString(PyExc_ValueError, "file descriptor must be readable for native profiling");
+        PyErr_SetString(PyExc_ValueError, "file descriptor must be readable");
         return NULL;
     }
 
@@ -330,21 +230,15 @@
 }
 
 static PyObject *
-disable_vmprof(PyObject *module, PyObject *args)
+disable_vmprof(PyObject *module, PyObject *noargs)
 {
-    int only_needed = 0;
-
-    if (!PyArg_ParseTuple(args, "|i", &only_needed))
-        return NULL;
-
-    if (emit_all_code_objects_helper(only_needed, 1))
-        return NULL;
-
     if (vmprof_disable() < 0) {
         PyErr_SetFromErrno(PyExc_OSError);
         return NULL;
     }
 
+    is_enabled = 0;
+
     if (PyErr_Occurred())
         return NULL;
 
@@ -352,23 +246,18 @@
 }
 
 static PyObject *
-write_all_code_objects(PyObject *module, PyObject *args)
+write_all_code_objects(PyObject *module, PyObject * seen_code_ids)
 {
-    int only_needed = 0;
-
-    if (!PyArg_ParseTuple(args, "|i", &only_needed))
-        return NULL;
-
-    if (emit_all_code_objects_helper(only_needed, 0))
-        return NULL;
+    // assumptions: signals must be disabled (see stop_sampling)
+    emit_all_code_objects(seen_code_ids);
 
     if (PyErr_Occurred())
         return NULL;
-
     Py_RETURN_NONE;
 }
 
 
+
 static PyObject *
 sample_stack_now(PyObject *module, PyObject * args)
 {
@@ -447,7 +336,6 @@
     if (o_srcfile == NULL) goto error;
     //
     return PyTuple_Pack(3, o_name, o_lineno, o_srcfile);
-
 error:
     Py_XDECREF(o_name);
     Py_XDECREF(o_lineno);
@@ -457,6 +345,20 @@
 }
 #endif
 
+static PyObject *
+stop_sampling(PyObject *module, PyObject *noargs)
+{
+    vmprof_ignore_signals(1);
+    return PyLong_NEW(vmp_profile_fileno());
+}
+
+static PyObject *
+start_sampling(PyObject *module, PyObject *noargs)
+{
+    vmprof_ignore_signals(0);
+    Py_RETURN_NONE;
+}
+
 #ifdef VMPROF_UNIX
 static PyObject * vmp_get_profile_path(PyObject *module, PyObject *noargs) {
     PyObject * o;
@@ -523,23 +425,30 @@
 }
 #endif
 
-
 static PyMethodDef VMProfMethods[] = {
     {"enable",  enable_vmprof, METH_VARARGS, "Enable profiling."},
-    {"disable", disable_vmprof, METH_VARARGS, "Disable profiling."},
-    {"write_all_code_objects", write_all_code_objects, METH_VARARGS,
-     "Write eagerly all the IDs of code objects"},
-    {"sample_stack_now", sample_stack_now, METH_VARARGS, "Sample the stack now"},
-    {"is_enabled", vmp_is_enabled, METH_NOARGS, "Indicates if vmprof is currently sampling."},
+    {"disable", disable_vmprof, METH_NOARGS, "Disable profiling."},
+    {"write_all_code_objects", write_all_code_objects, METH_O,
+        "Write eagerly all the IDs of code objects"},
+    {"sample_stack_now", sample_stack_now, METH_VARARGS,
+        "Sample the stack now"},
+    {"is_enabled", vmp_is_enabled, METH_NOARGS,
+        "Indicates if vmprof is currently sampling."},
+    {"stop_sampling", stop_sampling, METH_NOARGS,
+        "Blocks signals to occur and returns the file descriptor"},
+    {"start_sampling", start_sampling, METH_NOARGS,
+        "Unblocks vmprof signals. After compeltion vmprof will sample again"},
 #ifdef VMP_SUPPORTS_NATIVE_PROFILING
-    {"resolve_addr", resolve_addr, METH_VARARGS, "Return the name of the addr"},
+    {"resolve_addr", resolve_addr, METH_VARARGS,
+        "Returns the name of the given address"},
 #endif
 #ifdef VMPROF_UNIX
-    {"get_profile_path", vmp_get_profile_path, METH_NOARGS, "Profile path the profiler logs to."},
+    {"get_profile_path", vmp_get_profile_path, METH_NOARGS,
+        "Profile path the profiler logs to."},
     {"insert_real_time_thread", insert_real_time_thread, METH_NOARGS,
-     "Insert a thread into the real time profiling list."},
+        "Insert a thread into the real time profiling list."},
     {"remove_real_time_thread", remove_real_time_thread, METH_NOARGS,
-     "Remove a thread from the real time profiling list."},
+        "Remove a thread from the real time profiling list."},
 #endif
     {NULL, NULL, 0, NULL}        /* Sentinel */
 };
diff --git a/rpython/rlib/rvmprof/src/shared/compat.c b/rpython/rlib/rvmprof/src/shared/compat.c
--- a/rpython/rlib/rvmprof/src/shared/compat.c
+++ b/rpython/rlib/rvmprof/src/shared/compat.c
@@ -87,7 +87,9 @@
     }
     buf.tv_sec = tv.tv_sec;
     buf.tv_usec = tv.tv_usec;
-    strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
+    // IF we really support time zones:
+    // use a cross platform datetime library that outputs iso8601 strings
+    // strncpy(((char*)buffer)+__SIZE-8, tm.tm_zone, 8);
 
     buffer[0] = marker;
     (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
@@ -128,8 +130,9 @@
     buf.tv_usec = (system_time.wMilliseconds * 1000);
 
     // time zone not implemented on windows
+    // IF we really support time zones:
+    // use a cross platform datetime library that outputs iso8601 strings
     memset(((char*)buffer)+__SIZE-8, 0, 8);
-    (void)memcpy(((char*)buffer)+__SIZE-8, "UTC", 3);
 
     buffer[0] = marker;
     (void)memcpy(buffer+1, &buf, sizeof(struct timezone_buf));
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.c b/rpython/rlib/rvmprof/src/shared/symboltable.c
--- a/rpython/rlib/rvmprof/src/shared/symboltable.c
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.c
@@ -11,7 +11,6 @@
 
 #include <assert.h>
 #include <dlfcn.h>
-#include <errno.h>
 
 #if defined(VMPROF_LINUX)
 #include <link.h>
@@ -266,272 +265,3 @@
 #endif
     return 0;
 }
-
-#define WORD_SIZE sizeof(long)
-#define ADDR_SIZE sizeof(void*)
-#define MAXLEN 1024
-
-void _dump_native_symbol(int fileno, void * addr, char * sym, int linenumber, char * filename) {
-    char natsym[64];
-    off_t pos_before;
-    struct str {
-        void * addr;
-        // NOTE windows, not supported yet would be a problem for 64 bit
-        // hint: alignment
-        long size;
-        char str[1024];
-    } s;
-    fsync(fileno);
-    pos_before = lseek(fileno, 0, SEEK_CUR);
-    lseek(fileno, 0, SEEK_END);
-
-    s.addr = addr;
-    /* must mach '<lang>:<name>:<line>:<file>'
-     * 'n' has been chosen as lang here, because the symbol
-     * can be generated from several languages (e.g. C, C++, ...)
-     */
-    // MARKER_NATIVE_SYMBOLS is \x08
-    write(fileno, "\x08", 1);
-    if (sym == NULL || sym[0] == '\x00') {
-        snprintf(natsym, 64, "<native symbol %p>", addr);
-        sym = natsym;
-    }
-    if (filename != NULL) {
-        s.size = snprintf(s.str, 1024, "n:%s:%d:%s", sym, linenumber, filename);
-    } else {
-        s.size = snprintf(s.str, 1024, "n:%s:%d:-", sym, linenumber);
-    }
-    write(fileno, &s, sizeof(void*)+sizeof(long)+s.size);
-
-    lseek(fileno, pos_before, SEEK_SET);
-}
-
-static
-int cannot_read_profile = 0;
-
-ssize_t read_exactly(int fileno, void * buf, ssize_t size) {
-    assert(size >= 0 && "size parameter must be positive");
-
-    ssize_t r = 0;
-    if ((r = read(fileno, buf, (size_t)size)) == size) {
-        return r;
-    }
-
-    if (r == -1) {
-        while (errno == EINTR) {
-            if ((r = read(fileno, buf, (size_t)size)) == size) {
-                return r;
-            }
-        }
-    }
-
-    fprintf(stderr, "unhandled error in read_exactly. cannot proceed! could not read %d bytes", size);
-    cannot_read_profile = 1;
-    return -1;
-}
-
-int _skip_string(int fileno)
-{
-    long chars;
-    ssize_t count = read_exactly(fileno, &chars, sizeof(long));
-    //LOG("reading string of %d chars\n", chars);
-    if (count <= 0) {
-        return 1;
-    }
-    lseek(fileno, chars, SEEK_CUR);
-
-    return 0;
-}
-
-int _skip_header(int fileno, int * version, int * flags)
-{
-    unsigned char r[4];
-    (void)read_exactly(fileno, r, 4);
-    unsigned char count = r[3];
-    *version = (r[0] & 0xff) << 8 | (r[1] & 0xff);
-    *flags = r[2];
-    lseek(fileno, (int)count, SEEK_CUR);
-    return 0;
-}
-
-long _read_word(int fileno)
-{
-    long w;
-    (void)read_exactly(fileno, &w, WORD_SIZE);
-    return w;
-}
-
-void * _read_addr(int fileno)
-{
-    void * a;
-    (void)read_exactly(fileno, &a, ADDR_SIZE);
-    return a;
-}
-
-int _skip_word(int fileno)
-{
-    lseek(fileno, WORD_SIZE, SEEK_CUR);
-    return 0;
-}
-
-int _skip_addr(int fileno)
-{
-    lseek(fileno, ADDR_SIZE, SEEK_CUR);
-    return 0;
-}
-
-int _skip_time_and_zone(int fileno)
-{
-    lseek(fileno, sizeof(int64_t)*2 + 8, SEEK_CUR);
-    return 0;
-}
-
-KHASH_MAP_INIT_INT64(ptr, int)
-
-void vmp_scan_profile(int fileno, int dump_nat_sym, void *all_code_uids)
-{
-    off_t orig_pos, cur_pos;
-    char marker;
-    ssize_t count;
-    int version;
-    int flags;
-    int memory = 0, lines = 0, native = 0;
-    fsync(fileno);
-    orig_pos = lseek(fileno, 0, SEEK_CUR);
-
-    khash_t(ptr) * nat_syms = kh_init(ptr);
-    khiter_t it;
-
-    lseek(fileno, 5*WORD_SIZE, SEEK_SET);
-
-    while (1) {
-        if (cannot_read_profile == 1) {
-            cannot_read_profile = 0;
-            break;
-        }
-        count = read_exactly(fileno, &marker, 1);
-        if (count <= 0) {
-            break;
-        }
-        cur_pos = lseek(fileno, 0, SEEK_CUR);
-        LOG("posss 0x%llx %d\n", cur_pos-1, cur_pos-1);
-        switch (marker) {
-            case MARKER_HEADER: {
-                LOG("header 0x%llx\n", cur_pos);
-                if (_skip_header(fileno, &version, &flags) != 0) {
-                    kh_destroy(ptr, nat_syms);
-                    return;
-                }
-                memory = (flags & PROFILE_MEMORY) != 0;
-                native = (flags & PROFILE_NATIVE) != 0;
-                lines = (flags & PROFILE_LINES) != 0;
-                if (!native && dump_nat_sym) {
-                    lseek(fileno, 0, SEEK_END);
-                    kh_destroy(ptr, nat_syms);
-                    return;
-                }
-                break;
-            } case MARKER_META: {
-                LOG("meta 0x%llx\n", cur_pos);
-                if (_skip_string(fileno) != 0) { return; }
-                if (_skip_string(fileno) != 0) { return; }
-                break;
-            } case MARKER_TIME_N_ZONE:
-              case MARKER_TRAILER: {
-                LOG("tnz or trailer 0x%llx\n", cur_pos);
-                if (_skip_time_and_zone(fileno) != 0) { return; }
-                break;
-            } case MARKER_VIRTUAL_IP:
-              case MARKER_NATIVE_SYMBOLS: {
-                //LOG("virtip 0x%llx\n", cur_pos);
-                if (_skip_addr(fileno) != 0) { return; }
-                if (_skip_string(fileno) != 0) { return; }
-                break;
-            } case MARKER_STACKTRACE: {
-                long trace_count = _read_word(fileno);
-                long depth = _read_word(fileno);
-                long i;
-
-                LOG("stack 0x%llx %d %d\n", cur_pos, trace_count, depth);
-
-#ifdef RPYTHON_VMPROF
-                for (i = depth/2-1; i >= 0; i--) {
-                    long kind = (long)_read_addr(fileno);
-                    void * addr = _read_addr(fileno);
-                    if (kind == VMPROF_NATIVE_TAG) {
-#else
-                for (i = 0; i < depth; i++) {
-                    void * addr = _read_addr(fileno);
-                    if (lines && i % 2 == 0) {
-                        continue;
-                    }
-                    if (((intptr_t)addr & 0x1) == 1) {
-#endif
-                        /* dump the native symbol to disk */
-                        if (dump_nat_sym) {
-                            LOG("found kind %p\n", addr);
-
-                            // if the address has already been dumped,
-                            // do not log it again!
-                            it = kh_get(ptr, nat_syms, (khint64_t)addr);
-                            if (it == kh_end(nat_syms)) {
-                                char name[MAXLEN];
-                                char srcfile[MAXLEN];
-                                name[0] = 0;
-                                srcfile[0] = 0;
-                                int lineno = 0;
-                                if (vmp_resolve_addr(addr, name, MAXLEN, &lineno, srcfile, MAXLEN) == 0) {
-                                    LOG("dumping add %p, name %s, %s:%d\n", addr, name, srcfile, lineno);
-                                    _dump_native_symbol(fileno, addr, name, lineno, srcfile);
-                                    int ret;
-                                    it = kh_put(ptr, nat_syms, (khint64_t)addr, &ret);
-                                    kh_value(nat_syms, it) = 1;
-                                }
-                            }
-                        }
-#ifdef RPYTHON_VMPROF
-                    }
-#else
-                    } else {
-                        // cpython adds all addresses into a set to get the intersection
-                        // of all gc known code addresses
-                        if (all_code_uids != NULL) {
-                            PyObject *co_uid = PyLong_FromVoidPtr(addr);
-                            int check = PySet_Add(all_code_uids, co_uid);
-                            Py_CLEAR(co_uid);
-                        }
-                    }
-#endif
-                }
-                LOG("passed  memory %d \n", memory);
-
-                if (version >= VERSION_THREAD_ID) {
-                    if (_skip_addr(fileno) != 0) { return; } // thread id
-                }
-                if (memory) {
-                    if (_skip_addr(fileno) != 0) { return; } // profile memory
-                }
-
-                break;
-            } default: {
-                fprintf(stderr, "unknown marker 0x%x\n", marker);
-                lseek(fileno, 0, SEEK_END);
-                kh_destroy(ptr, nat_syms);
-                return;
-            }
-        }
-
-        cur_pos = lseek(fileno, 0, SEEK_CUR);
-        if (cur_pos >= orig_pos) {
-            break;
-        }
-    }
-
-    kh_destroy(ptr, nat_syms);
-    lseek(fileno, 0, SEEK_END);
-}
-
-void dump_native_symbols(int fileno)
-{
-    vmp_scan_profile(fileno, 1, NULL);
-}
diff --git a/rpython/rlib/rvmprof/src/shared/symboltable.h b/rpython/rlib/rvmprof/src/shared/symboltable.h
--- a/rpython/rlib/rvmprof/src/shared/symboltable.h
+++ b/rpython/rlib/rvmprof/src/shared/symboltable.h
@@ -2,22 +2,5 @@
 
 #define _GNU_SOURCE 1
 
-/**
- * Extract all the known symbols from the current process and
- * log them to the file descriptor. To read them see binary.py funcs:
- *
- * # encoded as a mapping
- * addr = read_word(fd); name = read_string(fd)
- *
- * A) It is not allowed to have two addresses (virtual ones are only valid
- * in the curent process) in this mapping to point to several symbols.
- * B) No duplicates are logged
- *
- * Deprecated, do not use
- */
-void dump_all_known_symbols(int fd);
-
-void dump_native_symbols(int fd);
-
 int vmp_resolve_addr(void * addr, char * name, int name_len, int * lineno,
                       char * srcfile, int srcfile_len);
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_common.h b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_common.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_common.h
@@ -39,7 +39,7 @@
 static inline ssize_t search_thread(pthread_t tid, ssize_t i) {
     if (i < 0)
         i = 0;
-    while (i < thread_count) {
+    while ((size_t)i < thread_count) {
         if (pthread_equal(threads[i], tid))
             return i;
         i++;
diff --git a/rpython/rlib/rvmprof/src/shared/vmprof_main.h b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
--- a/rpython/rlib/rvmprof/src/shared/vmprof_main.h
+++ b/rpython/rlib/rvmprof/src/shared/vmprof_main.h
@@ -166,7 +166,7 @@
     mythread_id = PyThread_get_thread_ident();
     istate = PyInterpreterState_Head();
     if (istate == NULL) {
-        fprintf(stderr, "WARNING: interp state head is null (for thread id %d)\n", mythread_id);
+        fprintf(stderr, "WARNING: interp state head is null (for thread id %ld)\n", mythread_id);
         return NULL;
     }
     // fish fish fish, it will NOT lock the keymutex in pythread
@@ -180,7 +180,7 @@
     } while ((istate = PyInterpreterState_Next(istate)) != NULL);
 
     // uh? not found?
-    fprintf(stderr, "WARNING: cannot find thread state (for thread id %d), sample will be thrown away\n", mythread_id);
+    fprintf(stderr, "WARNING: cannot find thread state (for thread id %ld), sample will be thrown away\n", mythread_id);
     return NULL;
 }
 #endif
@@ -462,13 +462,9 @@
 {
     int fileno = vmp_profile_fileno();
     fsync(fileno);
-    dump_native_symbols(fileno);
-
     (void)vmp_write_time_now(MARKER_TRAILER);
-
     teardown_rss();
 
-
     /* don't close() the file descriptor from here */
     vmp_set_profile_fileno(-1);
     return 0;
@@ -483,13 +479,16 @@
     disable_cpyprof();
 #endif
 
-    if (remove_sigprof_timer() == -1)
+    if (remove_sigprof_timer() == -1) {
         return -1;
-    if (remove_sigprof_handler() == -1)
+    }
+    if (remove_sigprof_handler() == -1) {
         return -1;
+    }
 #ifdef VMPROF_UNIX
-    if ((signal_type == SIGALRM) && remove_threads() == -1)
+    if ((signal_type == SIGALRM) && remove_threads() == -1) {
         return -1;
+    }
 #endif
     flush_codes();
     if (shutdown_concurrent_bufs(vmp_profile_fileno()) < 0)

From pypy.commits at gmail.com  Wed Jun  7 10:38:59 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Jun 2017 07:38:59 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: close branch
Message-ID: <59381003.5888df0a.f3680.0b8f@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91565:eaaa49735879
Date: 2017-06-07 10:38 -0400
http://bitbucket.org/pypy/pypy/changeset/eaaa49735879/

Log:	close branch


From pypy.commits at gmail.com  Wed Jun  7 10:41:53 2017
From: pypy.commits at gmail.com (plan_rich)
Date: Wed, 07 Jun 2017 07:41:53 -0700 (PDT)
Subject: [pypy-commit] pypy vmprof-0.4.8: document branch vmprof-0.4.8
Message-ID: <593810b1.12131c0a.dae96.3b74@mx.google.com>

Author: Richard Plangger <planrichi at gmail.com>
Branch: vmprof-0.4.8
Changeset: r91566:9f8a7fe9894a
Date: 2017-06-07 10:41 -0400
http://bitbucket.org/pypy/pypy/changeset/9f8a7fe9894a/

Log:	document branch vmprof-0.4.8

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,7 @@
 .. branch: cffi-char16-char32
 
 The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
+
+.. branch: vmprof-0.4.8
+
+Reduces the amount of C code needed to write native symbol information to the logfile.

From pypy.commits at gmail.com  Wed Jun  7 11:34:35 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 08:34:35 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: add 3.5 hashes
Message-ID: <59381d0b.5712190a.29bfc.3071@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r887:9220a439ce17
Date: 2017-06-07 18:34 +0300
http://bitbucket.org/pypy/pypy.org/changeset/9220a439ce17/

Log:	add 3.5 hashes

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -452,6 +452,12 @@
     ec1e34cc81a7f4086135bab29dcbe61d19fcd8d9d8fc1b149bea8373f94fd958  pypy2-v5.8.0-src.zip
     43d6217653e5bdc09e3ff8cb56fb52c4eb019429063d80107be4e88eef79ea8d  pypy2-v5.8.0-win32.zip
 
+pypy 3.5-v5.7.1 sha256::
+
+    9d090127335c3c0fd2b14c8835bf91752e62756e55ea06aad3353f24a6854223  pypy3-v5.8.0-src.tar.bz2
+    bb62f469df2421f854606c4181e66c14c58945180d03773020a4baf9a81c9fb1  pypy-c-jit-latest-linux64.tar.bz2
+    8c868b5c8d15ce8acdf967f3c25da44bf52f6c7aa1fd1e50ebd50590f98066a4  pypy3-v5.8.0-src.zip
+
 pypy2.7-v5.7.1 sha256::
 
     f125a227f8c814ba1698168a639ea6ca59bb69c280529639eed29076d8429a73  pypy2-v5.7.1-linux32.tar.bz2

From pypy.commits at gmail.com  Wed Jun  7 12:21:46 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 09:21:46 -0700 (PDT)
Subject: [pypy-commit] pypy default: add more highlights to the top-level of
 the release notice (hopefully accurate?)
Message-ID: <5938281a.8119190a.b5c2e.3126@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91567:68056ea67737
Date: 2017-06-07 19:21 +0300
http://bitbucket.org/pypy/pypy/changeset/68056ea67737/

Log:	add more highlights to the top-level of the release notice
	(hopefully accurate?)

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,6 +10,12 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
+We can now profile native frames in the vmprof_ statistical profiler, even in
+JITted code.
+
+The struct module functions `pack*` and `unpack*` are now much faster,
+especially on raw buffers and bytearrays.
+
 This release adds (but disables by default) link-time optimization and
 `profile guided optimization`_ of the base interpreter, which may make
 unjitted code run faster. To use these, translate with appropriate

From pypy.commits at gmail.com  Wed Jun  7 12:47:55 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 07 Jun 2017 09:47:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Don't preallocate RPython lists if we
 only know an upper bound on
Message-ID: <59382e3b.19512e0a.11a93.3294@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91568:6cbfa075de61
Date: 2017-06-07 18:47 +0200
http://bitbucket.org/pypy/pypy/changeset/6cbfa075de61/

Log:	Don't preallocate RPython lists if we only know an upper bound on
	the final size. See comments.

diff --git a/rpython/annotator/unaryop.py b/rpython/annotator/unaryop.py
--- a/rpython/annotator/unaryop.py
+++ b/rpython/annotator/unaryop.py
@@ -410,6 +410,7 @@
                 self.listdef.resize()
                 self.listdef.listitem.hint_maxlength = True
         elif 'fence' in hints:
+            self.listdef.resize()
             self = self.listdef.offspring(getbookkeeper())
         return self
 
diff --git a/rpython/translator/simplify.py b/rpython/translator/simplify.py
--- a/rpython/translator/simplify.py
+++ b/rpython/translator/simplify.py
@@ -1005,6 +1005,16 @@
 
         # - add a hint(vlist, iterable, {'maxlength'}) in the iterblock,
         #   where we can compute the known maximum length
+        # - new in June 2017: we do that only if 'exactlength' is True.
+        #   found some real use cases where the over-allocation scheme
+        #   was over-allocating far too much: the loop would only append
+        #   an item to the list after 'if some rare condition:'.  By
+        #   dropping this hint, we disable preallocation and cause the
+        #   append() to be done by checking the size, but still, after
+        #   the loop, we will turn the list into a fixed-size one.
+        #   ('maxlength_inexact' is never processed elsewhere; the hint
+        #   is still needed to prevent this function from being entered
+        #   in an infinite loop)
         link = iterblock.exits[0]
         vlist = self.contains_vlist(link.args)
         assert vlist
@@ -1014,7 +1024,8 @@
                 break
         else:
             raise AssertionError("lost 'iter' operation")
-        chint = Constant({'maxlength': True})
+        chint = Constant({'maxlength' if exactlength else 'maxlength_inexact':
+                          True})
         hint = op.hint(vlist, hlop.args[0], chint)
         iterblock.operations.append(hint)
         link.args = list(link.args)
diff --git a/rpython/translator/test/test_simplify.py b/rpython/translator/test/test_simplify.py
--- a/rpython/translator/test/test_simplify.py
+++ b/rpython/translator/test/test_simplify.py
@@ -336,11 +336,14 @@
         interp = LLInterpreter(t.rtyper)
         return interp, graph
 
-    def no_resize(self, graph):
+    def no_resize(self, graph, expect_resize=0):
+        found_resize = 0
         for block in graph.iterblocks():
             for op in block.operations:
                 if op.opname == 'direct_call':
-                    assert 'list_resize' not in repr(op.args[0])
+                    if 'list_resize' in repr(op.args[0]):
+                        found_resize += 1
+        assert found_resize == expect_resize
 
     def test_simple(self):
         def main(n):
@@ -367,7 +370,8 @@
         interp, graph = self.specialize(main, [int])
         res = interp.eval_graph(graph, [10])
         assert res == 5
-        self.no_resize(graph)
+        self.no_resize(graph, expect_resize=1)
+        # the non-exactness disables preallocating now, for sanity
 
     def test_mutated_after_listcomp(self):
         def main(n):
@@ -379,6 +383,7 @@
         assert res == 5 * 17
         res = interp.eval_graph(graph, [5])
         assert res == -42
+        self.no_resize(graph, expect_resize=1)   # after the loop
 
     def test_two_loops(self):
         def main(n, m):
@@ -397,6 +402,7 @@
         interp, graph = self.specialize(main, [int, int])
         res = interp.eval_graph(graph, [8, 3])
         assert res == 28 - 3
+        self.no_resize(graph)
 
     def test_dict(self):
         def main(n, m):
@@ -408,6 +414,7 @@
         assert res == 2 + 8 * 17 + 5 * 17
         res = interp.eval_graph(graph, [4, 4])
         assert res == 1 + 4 * 17 + 4 * 17
+        self.no_resize(graph)
 
 
     def test_list_iterator(self):
@@ -452,3 +459,4 @@
         interp, graph = self.specialize(main, [int])
         res = interp.eval_graph(graph, [10])
         assert res == 5 * 17
+        self.no_resize(graph)

From pypy.commits at gmail.com  Wed Jun  7 13:05:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 10:05:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: mention Gambit and actual struct speed
 improvement
Message-ID: <59383240.04c6190a.81847.3378@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91569:ad3681ed2701
Date: 2017-06-07 20:03 +0300
http://bitbucket.org/pypy/pypy/changeset/ad3681ed2701/

Log:	mention Gambit and actual struct speed improvement

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,11 +10,12 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-We can now profile native frames in the vmprof_ statistical profiler, even in
-JITted code.
+We added native PyPy support to profile frames in the vmprof_ statistical
+profiler.
 
-The struct module functions `pack*` and `unpack*` are now much faster,
-especially on raw buffers and bytearrays.
+The `struct`` module functions `pack*` and `unpack*` are now much faster,
+especially on raw buffers and bytearrays. Microbenchmarks show a 2x to 10x
+speedup. Thanks to Gambit_ for sponsoring this work.
 
 This release adds (but disables by default) link-time optimization and
 `profile guided optimization`_ of the base interpreter, which may make
@@ -60,6 +61,7 @@
 .. _`help`: project-ideas.html
 .. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+.. _Gambit: http://gambitresearch.com
 
 What is PyPy?
 =============

From pypy.commits at gmail.com  Wed Jun  7 13:09:47 2017
From: pypy.commits at gmail.com (antocuni)
Date: Wed, 07 Jun 2017 10:09:47 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix the name of Gambit Research
Message-ID: <5938335b.8a27190a.25486.117f@mx.google.com>

Author: Antonio Cuni <anto.cuni at gmail.com>
Branch: 
Changeset: r91570:3873f54fed26
Date: 2017-06-07 19:09 +0200
http://bitbucket.org/pypy/pypy/changeset/3873f54fed26/

Log:	fix the name of Gambit Research

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -15,7 +15,7 @@
 
 The `struct`` module functions `pack*` and `unpack*` are now much faster,
 especially on raw buffers and bytearrays. Microbenchmarks show a 2x to 10x
-speedup. Thanks to Gambit_ for sponsoring this work.
+speedup. Thanks to `Gambit Research`_ for sponsoring this work.
 
 This release adds (but disables by default) link-time optimization and
 `profile guided optimization`_ of the base interpreter, which may make

From pypy.commits at gmail.com  Wed Jun  7 13:13:02 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 10:13:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix link
Message-ID: <5938341e.0999190a.437df.35d0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91571:f3d63bdc598d
Date: 2017-06-07 20:12 +0300
http://bitbucket.org/pypy/pypy/changeset/f3d63bdc598d/

Log:	fix link

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -61,7 +61,7 @@
 .. _`help`: project-ideas.html
 .. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
-.. _Gambit: http://gambitresearch.com
+.. _`Gambit Reseach`: http://gambitresearch.com
 
 What is PyPy?
 =============

From pypy.commits at gmail.com  Wed Jun  7 13:31:22 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 10:31:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: mention the shadowstack issue that
 motivated the early release, other tweaks
Message-ID: <5938386a.55502e0a.46a1f.087b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91572:325ce0ce6e7d
Date: 2017-06-07 20:30 +0300
http://bitbucket.org/pypy/pypy/changeset/325ce0ce6e7d/

Log:	mention the shadowstack issue that motivated the early release,
	other tweaks

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -10,6 +10,10 @@
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
+We fixed critical bugs in the shadowstack_ rootfinder garbage collector
+strategy that crashed multithreaded programs and very rarely showed up
+even in single threaded programs.
+
 We added native PyPy support to profile frames in the vmprof_ statistical
 profiler.
 
@@ -20,7 +24,7 @@
 This release adds (but disables by default) link-time optimization and
 `profile guided optimization`_ of the base interpreter, which may make
 unjitted code run faster. To use these, translate with appropriate
-`options`_.  Be aware of `[1]`_, though.
+`options`_.  Be aware of `issues with gcc toolchains`_, though.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
@@ -52,7 +56,8 @@
 with making RPython's JIT even better.
 
 .. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
-.. _`[1]`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
+.. _shadowstack: config/translation.gcrootfinder.html
+.. _`issues with gcc toolchains`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html

From pypy.commits at gmail.com  Wed Jun  7 13:43:54 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 07 Jun 2017 10:43:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: formatting, fix links, tweak wording
Message-ID: <59383b5a.91092e0a.b60c9.346b@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91573:ff44fbcd839d
Date: 2017-06-07 20:43 +0300
http://bitbucket.org/pypy/pypy/changeset/ff44fbcd839d/

Log:	formatting, fix links, tweak wording

diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -17,7 +17,7 @@
 We added native PyPy support to profile frames in the vmprof_ statistical
 profiler.
 
-The `struct`` module functions `pack*` and `unpack*` are now much faster,
+The ``struct`` module functions ``pack*`` and ``unpack*`` are now much faster,
 especially on raw buffers and bytearrays. Microbenchmarks show a 2x to 10x
 speedup. Thanks to `Gambit Research`_ for sponsoring this work.
 
@@ -29,20 +29,18 @@
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
-Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
-release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
-as `these benchmarks show`_ it already gives a nice speed bump.
-We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
-"PyPy3.5" supports the Python 3.5 language).
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy and PEP 489 was fully implemented. Of course the
+bug fixes and performance enhancements mentioned above are part of both PyPy
+2.7 and PyPy 3.5.
 
 CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.10.1,
 improving an already great package for interfacing with C.
 
-As always, this release fixed many issues and bugs raised by the
+As always, this release fixed many other issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
 
-You can download the v5.8 release here:
+You can download the v5.8 releases here:
 
     http://pypy.org/download.html
 
@@ -57,6 +55,7 @@
 
 .. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
 .. _shadowstack: config/translation.gcrootfinder.html
+.. _vmprof: http://vmprof.readthedocs.io
 .. _`issues with gcc toolchains`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
@@ -66,7 +65,7 @@
 .. _`help`: project-ideas.html
 .. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
-.. _`Gambit Reseach`: http://gambitresearch.com
+.. _`Gambit Research`: http://gambitresearch.com
 
 What is PyPy?
 =============

From pypy.commits at gmail.com  Thu Jun  8 12:05:54 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Jun 2017 09:05:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: -A tests should not import random
 pypy.module.XXX (partial backout of 421d7a728d21)
Message-ID: <593975e2.478fdf0a.89a75.871f@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91574:f7455164661f
Date: 2017-06-08 17:05 +0100
http://bitbucket.org/pypy/pypy/changeset/f7455164661f/

Log:	-A tests should not import random pypy.module.XXX (partial backout
	of 421d7a728d21)

diff --git a/pypy/tool/pytest/apptest.py b/pypy/tool/pytest/apptest.py
--- a/pypy/tool/pytest/apptest.py
+++ b/pypy/tool/pytest/apptest.py
@@ -22,6 +22,14 @@
 pypyroot = os.path.dirname(pypydir)
 
 
+RENAMED_USEMODULES = {
+    '_winreg': 'winreg',
+    'exceptions': 'builtins',
+    'struct': '_struct',
+    'thread': '_thread',
+    'operator': '_operator',
+    'signal': '_signal'}
+
 class AppError(Exception):
     def __init__(self, excinfo):
         self.excinfo = excinfo
@@ -57,8 +65,7 @@
 
 
 def _rename_module(name):
-    mod = __import__("pypy.module." + name, globals(), locals(), ['Module'])
-    return mod.Module.applevel_name or name
+    return str(RENAMED_USEMODULES.get(name, name))
 
 
 def run_with_python(python_, target_, usemodules, **definitions):

From pypy.commits at gmail.com  Thu Jun  8 14:01:25 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 08 Jun 2017 11:01:25 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Add _testmultiphase.c to cpyext
 tests and get parts of it working
Message-ID: <593990f5.81851c0a.c29c0.b318@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91575:d2a5ce70ff79
Date: 2017-06-08 19:00 +0100
http://bitbucket.org/pypy/pypy/changeset/d2a5ce70ff79/

Log:	Add _testmultiphase.c to cpyext tests and get parts of it working

diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h
--- a/pypy/module/cpyext/include/Python.h
+++ b/pypy/module/cpyext/include/Python.h
@@ -84,6 +84,7 @@
 #include "pyconfig.h"
 
 #include "object.h"
+#include "typeslots.h"
 #include "abstract.h"
 #include "pymath.h"
 #include "pyport.h"
diff --git a/pypy/module/cpyext/include/typeslots.h b/pypy/module/cpyext/include/typeslots.h
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/include/typeslots.h
@@ -0,0 +1,85 @@
+/* Do not renumber the file; these numbers are part of the stable ABI. */
+/* Disabled, see #10181 */
+#undef Py_bf_getbuffer
+#undef Py_bf_releasebuffer
+#define Py_mp_ass_subscript 3
+#define Py_mp_length 4
+#define Py_mp_subscript 5
+#define Py_nb_absolute 6
+#define Py_nb_add 7
+#define Py_nb_and 8
+#define Py_nb_bool 9
+#define Py_nb_divmod 10
+#define Py_nb_float 11
+#define Py_nb_floor_divide 12
+#define Py_nb_index 13
+#define Py_nb_inplace_add 14
+#define Py_nb_inplace_and 15
+#define Py_nb_inplace_floor_divide 16
+#define Py_nb_inplace_lshift 17
+#define Py_nb_inplace_multiply 18
+#define Py_nb_inplace_or 19
+#define Py_nb_inplace_power 20
+#define Py_nb_inplace_remainder 21
+#define Py_nb_inplace_rshift 22
+#define Py_nb_inplace_subtract 23
+#define Py_nb_inplace_true_divide 24
+#define Py_nb_inplace_xor 25
+#define Py_nb_int 26
+#define Py_nb_invert 27
+#define Py_nb_lshift 28
+#define Py_nb_multiply 29
+#define Py_nb_negative 30
+#define Py_nb_or 31
+#define Py_nb_positive 32
+#define Py_nb_power 33
+#define Py_nb_remainder 34
+#define Py_nb_rshift 35
+#define Py_nb_subtract 36
+#define Py_nb_true_divide 37
+#define Py_nb_xor 38
+#define Py_sq_ass_item 39
+#define Py_sq_concat 40
+#define Py_sq_contains 41
+#define Py_sq_inplace_concat 42
+#define Py_sq_inplace_repeat 43
+#define Py_sq_item 44
+#define Py_sq_length 45
+#define Py_sq_repeat 46
+#define Py_tp_alloc 47
+#define Py_tp_base 48
+#define Py_tp_bases 49
+#define Py_tp_call 50
+#define Py_tp_clear 51
+#define Py_tp_dealloc 52
+#define Py_tp_del 53
+#define Py_tp_descr_get 54
+#define Py_tp_descr_set 55
+#define Py_tp_doc 56
+#define Py_tp_getattr 57
+#define Py_tp_getattro 58
+#define Py_tp_hash 59
+#define Py_tp_init 60
+#define Py_tp_is_gc 61
+#define Py_tp_iter 62
+#define Py_tp_iternext 63
+#define Py_tp_methods 64
+#define Py_tp_new 65
+#define Py_tp_repr 66
+#define Py_tp_richcompare 67
+#define Py_tp_setattr 68
+#define Py_tp_setattro 69
+#define Py_tp_str 70
+#define Py_tp_traverse 71
+#define Py_tp_members 72
+#define Py_tp_getset 73
+#define Py_tp_free 74
+#define Py_nb_matrix_multiply 75
+#define Py_nb_inplace_matrix_multiply 76
+#define Py_am_await 77
+#define Py_am_aiter 78
+#define Py_am_anext 79
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
+/* New in 3.5 */
+#define Py_tp_finalize 80
+#endif
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -289,6 +289,19 @@
     destructor tp_finalize;
 } PyTypeObject;
 
+typedef struct{
+    int slot;    /* slot id, see below */
+    void *pfunc; /* function pointer */
+} PyType_Slot;
+
+typedef struct{
+    const char* name;
+    int basicsize;
+    int itemsize;
+    unsigned int flags;
+    PyType_Slot *slots; /* terminated by slot==0. */
+} PyType_Spec;
+
 typedef struct _heaptypeobject {
     PyTypeObject ht_type;
     PyNumberMethods as_number;
diff --git a/pypy/module/cpyext/test/_testmultiphase.c b/pypy/module/cpyext/test/_testmultiphase.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/_testmultiphase.c
@@ -0,0 +1,627 @@
+/* Copied from CPython's Modules/_testmultiphase.c */
+/***************************************************/
+
+/* Testing module for multi-phase initialization of extension modules (PEP 489)
+ */
+
+#include "Python.h"
+
+///* Example objects */
+//typedef struct {
+//    PyObject_HEAD
+//    PyObject            *x_attr;        /* Attributes dictionary */
+//} ExampleObject;
+//
+///* Example methods */
+//
+//static int
+//Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+//{
+//    Py_VISIT(self->x_attr);
+//    return 0;
+//}
+//
+//static int
+//Example_finalize(ExampleObject *self)
+//{
+//    Py_CLEAR(self->x_attr);
+//    return 0;
+//}
+//
+//static PyObject *
+//Example_demo(ExampleObject *self, PyObject *args)
+//{
+//    PyObject *o = NULL;
+//    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+//        return NULL;
+//    if (o != NULL && PyUnicode_Check(o)) {
+//        Py_INCREF(o);
+//        return o;
+//    }
+//    Py_INCREF(Py_None);
+//    return Py_None;
+//}
+//
+//
+//static PyMethodDef Example_methods[] = {
+//    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+//        PyDoc_STR("demo() -> None")},
+//    {NULL,              NULL}           /* sentinel */
+//};
+//
+//static PyObject *
+//Example_getattro(ExampleObject *self, PyObject *name)
+//{
+//    if (self->x_attr != NULL) {
+//        PyObject *v = PyDict_GetItem(self->x_attr, name);
+//        if (v != NULL) {
+//            Py_INCREF(v);
+//            return v;
+//        }
+//    }
+//    return PyObject_GenericGetAttr((PyObject *)self, name);
+//}
+//
+//static int
+//Example_setattr(ExampleObject *self, char *name, PyObject *v)
+//{
+//    if (self->x_attr == NULL) {
+//        self->x_attr = PyDict_New();
+//        if (self->x_attr == NULL)
+//            return -1;
+//    }
+//    if (v == NULL) {
+//        int rv = PyDict_DelItemString(self->x_attr, name);
+//        if (rv < 0)
+//            PyErr_SetString(PyExc_AttributeError,
+//                "delete non-existing Example attribute");
+//        return rv;
+//    }
+//    else
+//        return PyDict_SetItemString(self->x_attr, name, v);
+//}
+//
+//static PyType_Slot Example_Type_slots[] = {
+//    {Py_tp_doc, "The Example type"},
+//    {Py_tp_finalize, Example_finalize},
+//    {Py_tp_traverse, Example_traverse},
+//    {Py_tp_getattro, Example_getattro},
+//    {Py_tp_setattr, Example_setattr},
+//    {Py_tp_methods, Example_methods},
+//    {0, 0},
+//};
+//
+//static PyType_Spec Example_Type_spec = {
+//    "_testimportexec.Example",
+//    sizeof(ExampleObject),
+//    0,
+//    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+//    Example_Type_slots
+//};
+
+/* Function of two integers returning integer */
+
+PyDoc_STRVAR(testexport_foo_doc,
+"foo(i,j)\n\
+\n\
+Return the sum of i and j.");
+
+static PyObject *
+testexport_foo(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:foo", &i, &j))
+        return NULL;
+    res = i + j;
+    return PyLong_FromLong(res);
+}
+
+/* Test that PyState registration fails  */
+/*
+PyDoc_STRVAR(call_state_registration_func_doc,
+"register_state(0): call PyState_FindModule()\n\
+register_state(1): call PyState_AddModule()\n\
+register_state(2): call PyState_RemoveModule()");
+
+static PyObject *
+call_state_registration_func(PyObject *mod, PyObject *args)
+{
+    int i, ret;
+    PyModuleDef *def = PyModule_GetDef(mod);
+    if (def == NULL) {
+        return NULL;
+    }
+    if (!PyArg_ParseTuple(args, "i:call_state_registration_func", &i))
+        return NULL;
+    switch (i) {
+        case 0:
+            mod = PyState_FindModule(def);
+            if (mod == NULL) {
+                Py_RETURN_NONE;
+            }
+            return mod;
+        case 1:
+            ret = PyState_AddModule(mod, def);
+            if (ret != 0) {
+                return NULL;
+            }
+            break;
+        case 2:
+            ret = PyState_RemoveModule(def);
+            if (ret != 0) {
+                return NULL;
+            }
+            break;
+    }
+    Py_RETURN_NONE;
+}
+*/
+
+//static PyType_Slot Str_Type_slots[] = {
+//    {Py_tp_base, NULL}, /* filled out in module exec function */
+//    {0, 0},
+//};
+//
+//static PyType_Spec Str_Type_spec = {
+//    "_testimportexec.Str",
+//    0,
+//    0,
+//    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+//    Str_Type_slots
+//};
+
+static PyMethodDef testexport_methods[] = {
+    {"foo",             testexport_foo,         METH_VARARGS,
+        testexport_foo_doc},
+/*    {"call_state_registration_func",  call_state_registration_func,
+        METH_VARARGS, call_state_registration_func_doc},*/
+    {NULL,              NULL}           /* sentinel */
+};
+
+//static int execfunc(PyObject *m)
+//{
+//    PyObject *temp = NULL;
+//
+//    /* Due to cross platform compiler issues the slots must be filled
+//     * here. It's required for portability to Windows without requiring
+//     * C++. */
+//    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+//
+//    /* Add a custom type */
+//    temp = PyType_FromSpec(&Example_Type_spec);
+//    if (temp == NULL)
+//        goto fail;
+//    if (PyModule_AddObject(m, "Example", temp) != 0)
+//        goto fail;
+//
+//    /* Add an exception type */
+//    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+//    if (temp == NULL)
+//        goto fail;
+//    if (PyModule_AddObject(m, "error", temp) != 0)
+//        goto fail;
+//
+//    /* Add Str */
+//    temp = PyType_FromSpec(&Str_Type_spec);
+//    if (temp == NULL)
+//        goto fail;
+//    if (PyModule_AddObject(m, "Str", temp) != 0)
+//        goto fail;
+//
+//    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+//        goto fail;
+//
+//    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+//        goto fail;
+//
+//    return 0;
+// fail:
+//    return -1;
+//}
+
+/* Helper for module definitions; there'll be a lot of them */
+#define TEST_MODULE_DEF(name, slots, methods) { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */ \
+    name,                                       /* m_name */ \
+    PyDoc_STR("Test module " name),             /* m_doc */ \
+    0,                                          /* m_size */ \
+    methods,                                    /* m_methods */ \
+    slots,                                      /* m_slots */ \
+    NULL,                                       /* m_traverse */ \
+    NULL,                                       /* m_clear */ \
+    NULL,                                       /* m_free */ \
+}
+
+PyModuleDef_Slot main_slots[] = {
+    //{Py_mod_exec, execfunc},
+    {0, NULL},
+};
+
+static PyModuleDef main_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+
+/**** Importing a non-module object ****/
+
+//static PyModuleDef def_nonmodule;
+//static PyModuleDef def_nonmodule_with_methods;
+//
+///* Create a SimpleNamespace(three=3) */
+//static PyObject*
+//createfunc_nonmodule(PyObject *spec, PyModuleDef *def)
+//{
+//    PyObject *dct, *ns, *three;
+//
+//    if (def != &def_nonmodule && def != &def_nonmodule_with_methods) {
+//        PyErr_SetString(PyExc_SystemError, "def does not match");
+//        return NULL;
+//    }
+//
+//    dct = PyDict_New();
+//    if (dct == NULL)
+//        return NULL;
+//
+//    three = PyLong_FromLong(3);
+//    if (three == NULL) {
+//        Py_DECREF(dct);
+//        return NULL;
+//    }
+//    PyDict_SetItemString(dct, "three", three);
+//    Py_DECREF(three);
+//
+//    ns = _PyNamespace_New(dct);
+//    Py_DECREF(dct);
+//    return ns;
+//}
+//
+//static PyModuleDef_Slot slots_create_nonmodule[] = {
+//    {Py_mod_create, createfunc_nonmodule},
+//    {0, NULL},
+//};
+//
+//static PyModuleDef def_nonmodule = TEST_MODULE_DEF(
+//    "_testmultiphase_nonmodule", slots_create_nonmodule, NULL);
+//
+//PyMODINIT_FUNC
+//PyInit__testmultiphase_nonmodule(PyObject *spec)
+//{
+//    return PyModuleDef_Init(&def_nonmodule);
+//}
+/*
+PyDoc_STRVAR(nonmodule_bar_doc,
+"bar(i,j)\n\
+\n\
+Return the difference of i - j.");
+
+static PyObject *
+nonmodule_bar(PyObject *self, PyObject *args)
+{
+    long i, j;
+    long res;
+    if (!PyArg_ParseTuple(args, "ll:bar", &i, &j))
+        return NULL;
+    res = i - j;
+    return PyLong_FromLong(res);
+}
+*/
+//static PyMethodDef nonmodule_methods[] = {
+//    {"bar", nonmodule_bar, METH_VARARGS, nonmodule_bar_doc},
+//    {NULL, NULL}           /* sentinel */
+//};
+//
+//static PyModuleDef def_nonmodule_with_methods = TEST_MODULE_DEF(
+//    "_testmultiphase_nonmodule_with_methods", slots_create_nonmodule, nonmodule_methods);
+//
+//PyMODINIT_FUNC
+//PyInit__testmultiphase_nonmodule_with_methods(PyObject *spec)
+//{
+//    return PyModuleDef_Init(&def_nonmodule_with_methods);
+//}
+
+/**** Non-ASCII-named modules ****/
+
+static PyModuleDef def_nonascii_latin = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_latin",           /* m_name */
+    PyDoc_STR("Module named in Czech"),         /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU__testmultiphase_zkouka_naten_evc07gi8e(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_latin);
+}
+
+static PyModuleDef def_nonascii_kana = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "_testmultiphase_nonascii_kana",            /* m_name */
+    PyDoc_STR("Module named in Japanese"),      /* m_doc */
+    0,                                          /* m_size */
+    NULL,                                       /* m_methods */
+    NULL,                                       /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInitU_eckzbwbhc6jpgzcx415x(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_nonascii_kana);
+}
+
+/*** Module with a single-character name ***/
+
+PyMODINIT_FUNC
+PyInit_x(PyObject *spec)
+{
+    return PyModuleDef_Init(&main_def);
+}
+
+/**** Testing NULL slots ****/
+
+static PyModuleDef null_slots_def = TEST_MODULE_DEF(
+    "_testmultiphase_null_slots", NULL, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_null_slots(PyObject *spec)
+{
+    return PyModuleDef_Init(&null_slots_def);
+}
+
+/**** Problematic modules ****/
+
+static PyModuleDef_Slot slots_bad_large[] = {
+    {_Py_mod_LAST_SLOT + 1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_large = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_large", slots_bad_large, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_large(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_large);
+}
+
+static PyModuleDef_Slot slots_bad_negative[] = {
+    {-1, NULL},
+    {0, NULL},
+};
+
+static PyModuleDef def_bad_negative = TEST_MODULE_DEF(
+    "_testmultiphase_bad_slot_negative", slots_bad_negative, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_bad_slot_negative(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_bad_negative);
+}
+
+static PyModuleDef def_create_int_with_state = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "create_with_state",                        /* m_name */
+    PyDoc_STR("Not a PyModuleObject object, but requests per-module state"),
+    10,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    //slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_int_with_state(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_int_with_state);
+}
+
+
+static PyModuleDef def_negative_size = { \
+    PyModuleDef_HEAD_INIT,                      /* m_base */
+    "negative_size",                            /* m_name */
+    PyDoc_STR("PyModuleDef with negative m_size"),
+    -1,                                         /* m_size */
+    NULL,                                       /* m_methods */
+    NULL, //slots_create_nonmodule,                     /* m_slots */
+    NULL,                                       /* m_traverse */
+    NULL,                                       /* m_clear */
+    NULL,                                       /* m_free */
+};
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_negative_size(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_negative_size);
+}
+
+
+static PyModuleDef uninitialized_def = TEST_MODULE_DEF("main", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_uninitialized(PyObject *spec)
+{
+    return (PyObject*) &uninitialized_def;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_null(PyObject *spec)
+{
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return NULL;
+}
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_export_unreported_exception(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad export function");
+    return PyModuleDef_Init(&main_def);
+}
+
+static PyObject*
+createfunc_null(PyObject *spec, PyModuleDef *def)
+{
+    return NULL;
+}
+
+PyModuleDef_Slot slots_create_null[] = {
+    {Py_mod_create, createfunc_null},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_null = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_null, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_null(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_null);
+}
+
+static PyObject*
+createfunc_raise(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return NULL;
+}
+
+static PyModuleDef_Slot slots_create_raise[] = {
+    {Py_mod_create, createfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_raise = TEST_MODULE_DEF(
+    "_testmultiphase_create_null", slots_create_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_raise(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_raise);
+}
+
+static PyObject*
+createfunc_unreported_exception(PyObject *spec, PyModuleDef *def)
+{
+    PyErr_SetString(PyExc_SystemError, "bad create function");
+    return PyModule_New("foo");
+}
+
+static PyModuleDef_Slot slots_create_unreported_exception[] = {
+    {Py_mod_create, createfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_create_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_create_unreported_exception", slots_create_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_create_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_create_unreported_exception);
+}
+
+//static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = {
+//    {Py_mod_create, createfunc_nonmodule},
+//    {Py_mod_exec, execfunc},
+//    {0, NULL},
+//};
+//
+//static PyModuleDef def_nonmodule_with_exec_slots = TEST_MODULE_DEF(
+//    "_testmultiphase_nonmodule_with_exec_slots", slots_nonmodule_with_exec_slots, NULL);
+//
+//PyMODINIT_FUNC
+//PyInit__testmultiphase_nonmodule_with_exec_slots(PyObject *spec)
+//{
+//    return PyModuleDef_Init(&def_nonmodule_with_exec_slots);
+//}
+
+static int
+execfunc_err(PyObject *mod)
+{
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_err[] = {
+    {Py_mod_exec, execfunc_err},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_err = TEST_MODULE_DEF(
+    "_testmultiphase_exec_err", slots_exec_err, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_err(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_err);
+}
+
+static int
+execfunc_raise(PyObject *spec)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return -1;
+}
+
+static PyModuleDef_Slot slots_exec_raise[] = {
+    {Py_mod_exec, execfunc_raise},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_raise = TEST_MODULE_DEF(
+    "_testmultiphase_exec_raise", slots_exec_raise, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_raise(PyObject *mod)
+{
+    return PyModuleDef_Init(&def_exec_raise);
+}
+
+static int
+execfunc_unreported_exception(PyObject *mod)
+{
+    PyErr_SetString(PyExc_SystemError, "bad exec function");
+    return 0;
+}
+
+static PyModuleDef_Slot slots_exec_unreported_exception[] = {
+    {Py_mod_exec, execfunc_unreported_exception},
+    {0, NULL},
+};
+
+static PyModuleDef def_exec_unreported_exception = TEST_MODULE_DEF(
+    "_testmultiphase_exec_unreported_exception", slots_exec_unreported_exception, NULL);
+
+PyMODINIT_FUNC
+PyInit__testmultiphase_exec_unreported_exception(PyObject *spec)
+{
+    return PyModuleDef_Init(&def_exec_unreported_exception);
+}
+
+/*** Helper for imp test ***/
+
+static PyModuleDef imp_dummy_def = TEST_MODULE_DEF("imp_dummy", main_slots, testexport_methods);
+
+PyMODINIT_FUNC
+PyInit_imp_dummy(PyObject *spec)
+{
+    return PyModuleDef_Init(&imp_dummy_def);
+}
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -130,3 +130,18 @@
         """
         raises(SystemError, self.import_module, name='multiphase', body=body,
                init=init)
+
+    def test_module(self):
+        import sys
+        from importlib import machinery, util
+        NAME = '_testmultiphase'
+        module = self.import_module(name=NAME)
+        finder = machinery.FileFinder(None)
+        spec = util.find_spec(NAME)
+        assert spec
+        assert module.__name__ == NAME
+        #assert module.__file__ == spec.origin
+        assert module.__package__ == ''
+        raises(AttributeError, 'module.__path__')
+        assert module is sys.modules[NAME]
+        assert isinstance(module.__loader__, machinery.ExtensionFileLoader)

From pypy.commits at gmail.com  Thu Jun  8 17:18:22 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 08 Jun 2017 14:18:22 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: ppc hashes
Message-ID: <5939bf1e.8887df0a.aefac.e84a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r888:5f484804be55
Date: 2017-06-08 23:18 +0200
http://bitbucket.org/pypy/pypy.org/changeset/5f484804be55/

Log:	ppc hashes

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -451,6 +451,8 @@
     504c2d522595baf8775ae1045a217a2b120732537861d31b889d47c340b58bd5  pypy2-v5.8.0-src.tar.bz2
     ec1e34cc81a7f4086135bab29dcbe61d19fcd8d9d8fc1b149bea8373f94fd958  pypy2-v5.8.0-src.zip
     43d6217653e5bdc09e3ff8cb56fb52c4eb019429063d80107be4e88eef79ea8d  pypy2-v5.8.0-win32.zip
+    2e464bcbc8216e55bb2433ace712130244fd1f3fa78de0c0c98745fd8ff12b03  pypy2-v5.8.0-ppc64.tar.bz2
+    5746823904df74423376e0326046e1171df9693a6d4c95e8ce14ca83534bae72  pypy2-v5.8.0-ppc64le.tar.bz2
 
 pypy 3.5-v5.7.1 sha256::
 

From pypy.commits at gmail.com  Thu Jun  8 17:46:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 08 Jun 2017 14:46:04 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: remove old hashes and regenerate
Message-ID: <5939c59c.c2a0df0a.ad18f.794d@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r889:0cc5e60a8c01
Date: 2017-06-09 00:45 +0300
http://bitbucket.org/pypy/pypy.org/changeset/0cc5e60a8c01/

Log:	remove old hashes and regenerate

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -77,8 +77,8 @@
 <p>We provide binaries for x86, ARM, PPC and s390x running on different operating systems such as
 Linux, Mac OS X and Windows:</p>
 <ul class="simple">
-<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.7.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.7.1.html">what's new in PyPy2.7?</a>)</li>
-<li>the Python3.5 compatible beta quality release &#8212; <strong>PyPy3.5 v5.7.1</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.7.1.html">what's new in PyPy3.5?</a>).</li>
+<li>the Python2.7 compatible release &#8212; <strong>PyPy2.7 v5.8.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.8.0.html">what's new in PyPy2.7?</a>)</li>
+<li>the Python3.5 compatible beta quality release &#8212; <strong>PyPy3.5 v5.8.0</strong> &#8212; (<a class="reference external" href="http://doc.pypy.org/en/latest/release-v5.8.0.html">what's new in PyPy3.5?</a>).</li>
 <li>the Python2.7 Software Transactional Memory special release &#8212; <strong>PyPy-STM 2.5.1</strong> (Linux x86-64 only)</li>
 </ul>
 <ul class="download-menu simple">
@@ -117,36 +117,36 @@
 <li>or <a class="reference internal" href="#translate">translate</a> your own PyPy.</li>
 </ul>
 </div>
-<div class="section" id="python2-7-compatible-pypy-5-7-1">
-<span id="release"></span><h1>Python2.7 compatible PyPy 5.7.1</h1>
+<div class="section" id="python2-7-compatible-pypy-5-8-0">
+<span id="release"></span><h1>Python2.7 compatible PyPy 5.8.0</h1>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux32.tar.bz2">Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armhf-raspbian.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armhf-raring.tar.bz2">ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-linux-armel.tar.bz2">ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-osx64.tar.bz2">Mac OS X binary (64bit)</a></li>
 <li>FreeBSD x86 and x86_64: see <a class="reference external" href="http://www.freshports.org/lang/pypy">FreshPorts</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-win32.zip">Windows binary (32bit)</a> (you might need the VS 2008 runtime library
 installer <a class="reference external" href="http://www.microsoft.com/en-us/download/details.aspx?id=5582">vcredist_x86.exe</a>.)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-ppc64.tar.bz2">PowerPC PPC64 Linux binary (64bit big-endian, Fedora 20)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-ppc64le.tar.bz2">PowerPC PPC64le Linux binary (64bit little-endian, Fedora 21)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-s390x.tar.bz2">s390x Linux binary (tar.bz2 built on Redhat Linux 7.2)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.tar.bz2">Source (tar.bz2)</a>; <a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.zip">Source (zip)</a>.  See below for more about the sources.</li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
 </div>
-<div class="section" id="python-3-5-3-compatible-pypy3-5-v5-7">
-<h1>Python 3.5.3 compatible PyPy3.5 v5.7</h1>
+<div class="section" id="python-3-5-3-compatible-pypy3-5-v5-8">
+<h1>Python 3.5.3 compatible PyPy3.5 v5.8</h1>
 <p class="download-menu">Warning: PyPy3.5 is considered <strong>beta software.</strong>  All binaries
 are thus called &ldquo;beta&rdquo;.  It is known to be rarely much slower than
 PyPy 2.  You are welcome to use it anyway; if you're lucky it will
 be fast in your case.</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-src.tar.bz2">Source (tar.bz2)</a></li>
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.7.1-src.zip">Source (zip)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-linux64.tar.bz2">Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)</a> (see <tt class="docutils literal">[1]</tt> below)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-src.tar.bz2">Source (tar.bz2)</a></li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy3-v5.8.0-src.zip">Source (zip)</a></li>
 <li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads">All our downloads,</a> including previous versions.  We also have a
 <a class="reference external" href="http://buildbot.pypy.org/mirror/">mirror</a>, but please use only if you have troubles accessing the links above</li>
 </ul>
@@ -200,7 +200,7 @@
 uncompressed, they run in-place.  For now you can uncompress them
 either somewhere in your home directory or, say, in <tt class="docutils literal">/opt</tt>, and
 if you want, put a symlink from somewhere like
-<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.7.1/bin/pypy</span></tt>.  Do
+<tt class="docutils literal">/usr/local/bin/pypy</tt> to <tt class="docutils literal"><span class="pre">/path/to/pypy2-5.8.0/bin/pypy</span></tt>.  Do
 not move or copy the executable <tt class="docutils literal">pypy</tt> outside the tree &ndash; put
 a symlink to it, otherwise it will not find its libraries.</p>
 </div>
@@ -281,7 +281,7 @@
 <p>Alternatively, the following smaller package contains the source at
 the same revision as the above binaries:</p>
 <ul class="simple">
-<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.7.1-src.tar.bz2">pypy2-v5.7.1-src.tar.bz2</a> (sources)</li>
+<li><a class="reference external" href="https://bitbucket.org/pypy/pypy/downloads/pypy2-v5.8.0-src.tar.bz2">pypy2-v5.8.0-src.tar.bz2</a> (sources)</li>
 </ul>
 </li>
 <li><p class="first">Make sure you <strong>installed the dependencies.</strong>  See the list <a class="reference external" href="http://pypy.readthedocs.org/en/latest/build.html#install-build-time-dependencies">here</a>.</p>
@@ -396,26 +396,26 @@
 <div class="section" id="checksums">
 <h1>Checksums</h1>
 <p>Here are the checksums for each of the downloads</p>
-<p>pypy2.7-v5.7.1 sha256:</p>
+<p>pypy2.7-v5.8.0 sha256:</p>
 <pre class="literal-block">
-f125a227f8c814ba1698168a639ea6ca59bb69c280529639eed29076d8429a73  pypy2-v5.7.1-linux32.tar.bz2
-c4fa3da42156bed4a9d912433b618a141e0c5161d7cc8c328786736ea5d1c2da  pypy2-v5.7.1-linux64.tar.bz2
-591a4a73cc945a1125848f3615a28559692db8febf677d7087eaef40cb119a8d  pypy2-v5.7.1-linux-armel.tar.bz2
-c1b1a0968b22c58672f7492dc7900bc85e3bd02c791f219f31401a00ef387207  pypy2-v5.7.1-linux-armhf-raring.tar.bz2
-67544f8c4b284db71cf1af74edef290722f97f82476cbdaff2015fdab244c6ee  pypy2-v5.7.1-linux-armhf-raspbian.tar.bz2
-4e99ba356432861534917a9477ace0ccee617bd631512759a530f8383e153a3d  pypy2-v5.7.1-osx64.tar.bz2
-395c57a1aa078c9ae9aa9f18263bb91fa23114bae4d133d10560eaea30c51892  pypy2-v5.7.1-s390x.tar.bz2
-d01bee43c6df79f7bbc1149bb3e85f489491fb2358a6a1f9a7f0d6e07715832f  pypy2-v5.7.1-src.tar.bz2
-f580a7b41cc09e030bbff66752044bdfbc193ec617e575c64bf65c136c298076  pypy2-v5.7.1-src.zip
-a3ba7c946635236836f8536d8767a0f456b3b9a86876cb5c3173a04522bf451b  pypy2-v5.7.1-win32.zip
-a4edb857c1f001bab4b545f2f87f3653e8cd882a95233d60d584edb9700722e9  pypy2-v5.7.1-ppc64.tar.bz2
-5db9bb6429d9cf146f74360ec30b40bd0df78116f458cf14035a0b7daefa35eb  pypy2-v5.7.1-ppc64le.tar.bz2
+a0b125a5781f7e5ddfc3baca46503b14f4ee6a0e234e8d72bfcf3afdf4120bef  pypy2-v5.8.0-linux32.tar.bz2
+6274292d0e954a2609b15978cde6efa30942ba20aa5d2acbbf1c70c0a54e9b1e  pypy2-v5.8.0-linux64.tar.bz2
+28b7fd0cc7418ffc66c71520728e87941be40ebf4b82675c57e25598a2a702b0  pypy2-v5.8.0-linux-armel.tar.bz2
+ddceca9c5c9a456d4bf1beab177660adffbbdf255a922244e1cc05f20318be46  pypy2-v5.8.0-linux-armhf-raring.tar.bz2
+da58279a0e3706889fc0df06087cea08f8cfd22322139fe9bae73ef9b2d119b7  pypy2-v5.8.0-linux-armhf-raspbian.tar.bz2
+04b61d1cf13aaca6d0420e854c820b8bd049dc88be16c02542abe8ca26eb075c  pypy2-v5.8.0-osx64.tar.bz2
+35aea25e2b9d2f7c8742c47e4e7474ef0f93ce1b5e3d4f5a99795bab23c1ad2c  pypy2-v5.8.0-s390x.tar.bz2
+504c2d522595baf8775ae1045a217a2b120732537861d31b889d47c340b58bd5  pypy2-v5.8.0-src.tar.bz2
+ec1e34cc81a7f4086135bab29dcbe61d19fcd8d9d8fc1b149bea8373f94fd958  pypy2-v5.8.0-src.zip
+43d6217653e5bdc09e3ff8cb56fb52c4eb019429063d80107be4e88eef79ea8d  pypy2-v5.8.0-win32.zip
+2e464bcbc8216e55bb2433ace712130244fd1f3fa78de0c0c98745fd8ff12b03  pypy2-v5.8.0-ppc64.tar.bz2
+5746823904df74423376e0326046e1171df9693a6d4c95e8ce14ca83534bae72  pypy2-v5.8.0-ppc64le.tar.bz2
 </pre>
-<p>pypy 3.5-v5.7.1 sha256:</p>
+<p>pypy 3.5-v5.8.0 sha256:</p>
 <pre class="literal-block">
-2abaa54d88c9b70b64c37083e7e430a1d3a8f78f8de92e484a988b7aca1e50a7  pypy3-v5.7.1-linux64.tar.bz2
-40ece0145282980ac121390f13709404c0532896507d5767496381180b631bd0  pypy3-v5.7.1-src.tar.bz2
-97fb0075ab2ecfb912fab996c71c6a8475ff18766db6c9e060f9384b809500bd  pypy3-v5.7.1-src.zip
+9d090127335c3c0fd2b14c8835bf91752e62756e55ea06aad3353f24a6854223  pypy3-v5.8.0-src.tar.bz2
+bb62f469df2421f854606c4181e66c14c58945180d03773020a4baf9a81c9fb1  pypy-c-jit-latest-linux64.tar.bz2
+8c868b5c8d15ce8acdf967f3c25da44bf52f6c7aa1fd1e50ebd50590f98066a4  pypy3-v5.8.0-src.zip
 </pre>
 </div>
 </div>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -454,30 +454,9 @@
     2e464bcbc8216e55bb2433ace712130244fd1f3fa78de0c0c98745fd8ff12b03  pypy2-v5.8.0-ppc64.tar.bz2
     5746823904df74423376e0326046e1171df9693a6d4c95e8ce14ca83534bae72  pypy2-v5.8.0-ppc64le.tar.bz2
 
-pypy 3.5-v5.7.1 sha256::
+pypy 3.5-v5.8.0 sha256::
 
     9d090127335c3c0fd2b14c8835bf91752e62756e55ea06aad3353f24a6854223  pypy3-v5.8.0-src.tar.bz2
     bb62f469df2421f854606c4181e66c14c58945180d03773020a4baf9a81c9fb1  pypy-c-jit-latest-linux64.tar.bz2
     8c868b5c8d15ce8acdf967f3c25da44bf52f6c7aa1fd1e50ebd50590f98066a4  pypy3-v5.8.0-src.zip
 
-pypy2.7-v5.7.1 sha256::
-
-    f125a227f8c814ba1698168a639ea6ca59bb69c280529639eed29076d8429a73  pypy2-v5.7.1-linux32.tar.bz2
-    c4fa3da42156bed4a9d912433b618a141e0c5161d7cc8c328786736ea5d1c2da  pypy2-v5.7.1-linux64.tar.bz2
-    591a4a73cc945a1125848f3615a28559692db8febf677d7087eaef40cb119a8d  pypy2-v5.7.1-linux-armel.tar.bz2
-    c1b1a0968b22c58672f7492dc7900bc85e3bd02c791f219f31401a00ef387207  pypy2-v5.7.1-linux-armhf-raring.tar.bz2
-    67544f8c4b284db71cf1af74edef290722f97f82476cbdaff2015fdab244c6ee  pypy2-v5.7.1-linux-armhf-raspbian.tar.bz2
-    4e99ba356432861534917a9477ace0ccee617bd631512759a530f8383e153a3d  pypy2-v5.7.1-osx64.tar.bz2
-    395c57a1aa078c9ae9aa9f18263bb91fa23114bae4d133d10560eaea30c51892  pypy2-v5.7.1-s390x.tar.bz2
-    d01bee43c6df79f7bbc1149bb3e85f489491fb2358a6a1f9a7f0d6e07715832f  pypy2-v5.7.1-src.tar.bz2
-    f580a7b41cc09e030bbff66752044bdfbc193ec617e575c64bf65c136c298076  pypy2-v5.7.1-src.zip
-    a3ba7c946635236836f8536d8767a0f456b3b9a86876cb5c3173a04522bf451b  pypy2-v5.7.1-win32.zip
-    a4edb857c1f001bab4b545f2f87f3653e8cd882a95233d60d584edb9700722e9  pypy2-v5.7.1-ppc64.tar.bz2
-    5db9bb6429d9cf146f74360ec30b40bd0df78116f458cf14035a0b7daefa35eb  pypy2-v5.7.1-ppc64le.tar.bz2
-
-
-pypy 3.5-v5.7.1 sha256::
-
-    2abaa54d88c9b70b64c37083e7e430a1d3a8f78f8de92e484a988b7aca1e50a7  pypy3-v5.7.1-linux64.tar.bz2
-    40ece0145282980ac121390f13709404c0532896507d5767496381180b631bd0  pypy3-v5.7.1-src.tar.bz2
-    97fb0075ab2ecfb912fab996c71c6a8475ff18766db6c9e060f9384b809500bd  pypy3-v5.7.1-src.zip

From pypy.commits at gmail.com  Fri Jun  9 06:40:21 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Jun 2017 03:40:21 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: fix for proper 3.5 package name
Message-ID: <593a7b15.12131c0a.9e527.6922@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: extradoc
Changeset: r890:3e1bcf5c6a4d
Date: 2017-06-09 13:39 +0300
http://bitbucket.org/pypy/pypy.org/changeset/3e1bcf5c6a4d/

Log:	fix for proper 3.5 package name

diff --git a/download.html b/download.html
--- a/download.html
+++ b/download.html
@@ -414,7 +414,7 @@
 <p>pypy 3.5-v5.8.0 sha256:</p>
 <pre class="literal-block">
 9d090127335c3c0fd2b14c8835bf91752e62756e55ea06aad3353f24a6854223  pypy3-v5.8.0-src.tar.bz2
-bb62f469df2421f854606c4181e66c14c58945180d03773020a4baf9a81c9fb1  pypy-c-jit-latest-linux64.tar.bz2
+57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035  pypy3-v5.8.0-linux64.tar.bz2
 8c868b5c8d15ce8acdf967f3c25da44bf52f6c7aa1fd1e50ebd50590f98066a4  pypy3-v5.8.0-src.zip
 </pre>
 </div>
diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -457,6 +457,6 @@
 pypy 3.5-v5.8.0 sha256::
 
     9d090127335c3c0fd2b14c8835bf91752e62756e55ea06aad3353f24a6854223  pypy3-v5.8.0-src.tar.bz2
-    bb62f469df2421f854606c4181e66c14c58945180d03773020a4baf9a81c9fb1  pypy-c-jit-latest-linux64.tar.bz2
+    57d871a7f1135719c138cee4e3533c3275d682a76a40ff668e95150c65923035  pypy3-v5.8.0-linux64.tar.bz2
     8c868b5c8d15ce8acdf967f3c25da44bf52f6c7aa1fd1e50ebd50590f98066a4  pypy3-v5.8.0-src.zip
 

From pypy.commits at gmail.com  Fri Jun  9 06:48:04 2017
From: pypy.commits at gmail.com (mattip)
Date: Fri, 09 Jun 2017 03:48:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: make more noise when something goes
 wrong with the repackaging
Message-ID: <593a7ce4.b48ddf0a.8d5cf.3b20@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91576:927530bd5d2c
Date: 2017-06-09 13:47 +0300
http://bitbucket.org/pypy/pypy/changeset/927530bd5d2c/

Log:	make more noise when something goes wrong with the repackaging

diff --git a/pypy/tool/release/repackage.sh b/pypy/tool/release/repackage.sh
--- a/pypy/tool/release/repackage.sh
+++ b/pypy/tool/release/repackage.sh
@@ -36,7 +36,10 @@
     hgcheck=`tar -tf pypy-c-jit-latest-$plat.tar.bz2 |head -n1 | cut -d- -f5`
     if [ "$hgcheck" != "$hgrev" ]
     then
+        echo xxxxxxxxxxxxxxxxxxxxxx
         echo $plat hg tag mismatch, expected $hgrev, got $hgcheck
+        echo xxxxxxxxxxxxxxxxxxxxxx
+        rm pypy-c-jit-latest-$plat.tar.bz2
         continue
     fi
     tar -xf pypy-c-jit-latest-$plat.tar.bz2

From pypy.commits at gmail.com  Fri Jun  9 16:44:46 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 09 Jun 2017 13:44:46 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <593b08be.0d451c0a.a7179.47ea@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r891:bb5e71b4d612
Date: 2017-06-09 22:44 +0200
http://bitbucket.org/pypy/pypy.org/changeset/bb5e71b4d612/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -15,7 +15,7 @@
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66846 of $105000 (63.7%)
+   $66891 of $105000 (63.7%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2648 left
+    we have $2688 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>
diff --git a/don4.html b/don4.html
--- a/don4.html
+++ b/don4.html
@@ -17,7 +17,7 @@
    2nd call:
    <!-- Income:PyPy:Donations:Transactional Memory from April 8, 2014,
    divided by 0.9 -->
-   $59060 of $80000 (73.8%)
+   $59070 of $80000 (73.8%)
    <div id="progressbar">
    </div>
 
@@ -29,7 +29,7 @@
   
   <!--
   Current status:
-    we have $48246 left
+    we have $48255 left
   in the account.</br> <a href="tmdonate2.html">Read proposal (2nd call)</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Jun  9 17:49:27 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 09 Jun 2017 14:49:27 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Fix nested
 measurements of wait time during transaction start
Message-ID: <593b17e7.430d1c0a.37f3d.7bf7@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2066:e425b6d2a050
Date: 2017-06-09 18:09 +0200
http://bitbucket.org/pypy/stmgc/changeset/e425b6d2a050/

Log:	Fix nested measurements of wait time during transaction start

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1073,14 +1073,15 @@
 
 static void _do_start_transaction(stm_thread_local_t *tl)
 {
-    start_timer();
-
     assert(!_stm_in_transaction(tl));
     tl->wait_event_emitted = 0;
 
     acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
+    // acquiring segment is measured as wait time
+    start_timer();
+
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
     assert(STM_PSEGMENT->transaction_state == TS_NONE);
     timing_event(tl, STM_TRANSACTION_START);
@@ -1127,10 +1128,12 @@
         rv++;        /* incr it but enter_safe_point_if_requested() aborted */
     STM_SEGMENT->transaction_read_version = rv;
 
+    pause_timer();
     /* Warning: this safe-point may run light finalizers and register
        commit/abort callbacks if a major GC is triggered here */
     enter_safe_point_if_requested();
     dprintf(("> start_transaction\n"));
+    continue_timer();
 
     s_mutex_unlock();   // XXX it's probably possible to not acquire this here
 
@@ -1149,6 +1152,8 @@
 
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
+    start_timer();
+
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
     long repeat_count = did_abort;    /* test/support.py */
@@ -1162,10 +1167,17 @@
             memcpy(tl->mem_reset_on_abort, tl->mem_stored_for_reset_on_abort,
                    tl->mem_bytes_to_reset_on_abort);
     }
+
+    // _do_start_transaction is instrumented as well and pauses for waits
+    pause_timer();
     _do_start_transaction(tl);
+    continue_timer();
 
     STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
                                         stm_get_transaction_length(tl));
+
+    stop_timer_and_publish(STM_DURATION_START_TRX);
+
     return repeat_count;
 }
 

From pypy.commits at gmail.com  Fri Jun  9 17:49:29 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 09 Jun 2017 14:49:29 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix nested
 measurement of waiting time and time in validation
Message-ID: <593b17e9.4c3a1c0a.71b6d.8b99@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-overheads-instrumentation
Changeset: r2067:39e259c124d6
Date: 2017-06-09 18:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/39e259c124d6/

Log:	Fix nested measurement of waiting time and time in validation

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -463,7 +463,9 @@
 #endif
 
     if (STM_PSEGMENT->last_commit_log_entry->next == INEV_RUNNING) {
-        wait_for_inevitable();
+        pause_timer();
+        wait_for_inevitable(); // TODO may abort!! timing event lost
+        continue_timer();
         goto retry_from_start;   /* redo _stm_validate() now */
     }
 

From pypy.commits at gmail.com  Fri Jun  9 17:49:31 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 09 Jun 2017 14:49:31 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix nested
 measurements of wait time during transaction start
Message-ID: <593b17eb.4eacdf0a.9a8a2.8152@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-overheads-instrumentation
Changeset: r2068:6161d44ff649
Date: 2017-06-09 18:58 +0200
http://bitbucket.org/pypy/stmgc/changeset/6161d44ff649/

Log:	Fix nested measurements of wait time during transaction start

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1072,14 +1072,15 @@
 
 static void _do_start_transaction(stm_thread_local_t *tl)
 {
-    start_timer();
-
     assert(!_stm_in_transaction(tl));
     tl->wait_event_emitted = 0;
 
     acquire_thread_segment(tl);
     /* GS invalid before this point! */
 
+    // acquiring segment is measured as wait time
+    start_timer();
+
     assert(STM_PSEGMENT->safe_point == SP_NO_TRANSACTION);
     assert(STM_PSEGMENT->transaction_state == TS_NONE);
     timing_event(tl, STM_TRANSACTION_START);
@@ -1126,10 +1127,12 @@
         rv++;        /* incr it but enter_safe_point_if_requested() aborted */
     STM_SEGMENT->transaction_read_version = rv;
 
+    pause_timer();
     /* Warning: this safe-point may run light finalizers and register
        commit/abort callbacks if a major GC is triggered here */
     enter_safe_point_if_requested();
     dprintf(("> start_transaction\n"));
+    continue_timer();
 
     s_mutex_unlock();   // XXX it's probably possible to not acquire this here
 
@@ -1148,6 +1151,8 @@
 
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
+    start_timer();
+
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
     long repeat_count = did_abort;    /* test/support.py */
@@ -1161,13 +1166,20 @@
             memcpy(tl->mem_reset_on_abort, tl->mem_stored_for_reset_on_abort,
                    tl->mem_bytes_to_reset_on_abort);
     }
+
+    // _do_start_transaction is instrumented as well and pauses for waits
+    pause_timer();
     _do_start_transaction(tl);
+    continue_timer();
 
     if (repeat_count == 0) {  /* else, 'nursery_mark' was already set
                                  in abort_data_structures_from_segment_num() */
         STM_SEGMENT->nursery_mark = ((stm_char *)_stm_nursery_start +
                                      stm_fill_mark_nursery_bytes);
     }
+
+    stop_timer_and_publish(STM_DURATION_START_TRX);
+
     return repeat_count;
 }
 

From pypy.commits at gmail.com  Fri Jun  9 17:49:33 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 09 Jun 2017 14:49:33 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Merge
 instrumentation bugfixes
Message-ID: <593b17ed.d78adf0a.e2006.52e9@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2069:2139691f1c0c
Date: 2017-06-09 20:42 +0200
http://bitbucket.org/pypy/stmgc/changeset/2139691f1c0c/

Log:	Merge instrumentation bugfixes


From pypy.commits at gmail.com  Sat Jun 10 05:45:07 2017
From: pypy.commits at gmail.com (tobweber)
Date: Sat, 10 Jun 2017 02:45:07 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Merge single
 thread mode events and fixes for timing macros
Message-ID: <593bbfa3.1c87df0a.f9df5.681c@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-overheads-instrumentation
Changeset: r2070:45d8e22cca0d
Date: 2017-06-10 11:43 +0200
http://bitbucket.org/pypy/stmgc/changeset/45d8e22cca0d/

Log:	Merge single thread mode events and fixes for timing macros

diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -29,20 +29,20 @@
 
 #define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &duration_data };                                     \
+        { .duration = &(duration_data) };                                     \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 
 #define publish_event(thread_local, event)                                  \
     (timing_enabled() ?                                                     \
-        stmcb_timing_event(thread_local, event, &stm_duration_payload) :    \
+        stmcb_timing_event((thread_local), (event), &stm_duration_payload) :\
         (void)0);
 
 #define stop_timer_and_publish_for_thread(thread_local, event)              \
     pause_timer()                                                           \
     stm_duration_payload(duration)                                          \
-    assert(thread_local != NULL);                                           \
-    publish_event(thread_local, event)
+    assert((thread_local) != NULL);                                         \
+    publish_event((thread_local), (event))
 
 #define stop_timer_and_publish(event)                                       \
-    stop_timer_and_publish_for_thread(STM_SEGMENT->running_thread, event)
+    stop_timer_and_publish_for_thread(STM_SEGMENT->running_thread, (event))
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -585,6 +585,10 @@
     STM_DURATION_MAJOR_GC_LOG_ONLY,
     STM_DURATION_MAJOR_GC_FULL,
 
+    STM_SINGLE_THREAD_MODE_ON,
+    STM_SINGLE_THREAD_MODE_OFF,
+    STM_SINGLE_THREAD_MODE_ADAPTIVE,
+
     _STM_EVENT_N
 };
 

From pypy.commits at gmail.com  Sun Jun 11 12:05:32 2017
From: pypy.commits at gmail.com (tobweber)
Date: Sun, 11 Jun 2017 09:05:32 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-adaptive-trx-length-per-thread: Merge
 transaction start instrumentation fix
Message-ID: <593d6a4c.66a8df0a.bf777.8387@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-adaptive-trx-length-per-thread
Changeset: r2072:0621fb926e22
Date: 2017-06-11 17:03 +0200
http://bitbucket.org/pypy/stmgc/changeset/0621fb926e22/

Log:	Merge transaction start instrumentation fix

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1152,8 +1152,6 @@
 
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
-    start_timer();
-
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
     long repeat_count = did_abort;    /* test/support.py */
@@ -1161,6 +1159,9 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
+
+    start_timer();
+
     if (repeat_count) {
         /* only if there was an abort, we need to reset the memory: */
         if (tl->mem_reset_on_abort)
@@ -1168,7 +1169,7 @@
                    tl->mem_bytes_to_reset_on_abort);
     }
 
-    // _do_start_transaction is instrumented as well and pauses for waits
+    // _do_start_transaction is instrumented as well b/c it needs to pause for waits
     pause_timer();
     _do_start_transaction(tl);
     continue_timer();
diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -29,7 +29,7 @@
 
 #define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &(duration_data) };                                     \
+        { .duration = &(duration_data) };                                   \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 

From pypy.commits at gmail.com  Sun Jun 11 12:05:30 2017
From: pypy.commits at gmail.com (tobweber)
Date: Sun, 11 Jun 2017 09:05:30 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-overheads-instrumentation: Fix wrong
 transaction start measurements when transaction was aborted with longjmp
Message-ID: <593d6a4a.81851c0a.7634.1227@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-overheads-instrumentation
Changeset: r2071:06b84e2a6b84
Date: 2017-06-11 16:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/06b84e2a6b84/

Log:	Fix wrong transaction start measurements when transaction was
	aborted with longjmp

diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -1151,8 +1151,6 @@
 
 long _stm_start_transaction(stm_thread_local_t *tl)
 {
-    start_timer();
-
     s_mutex_lock();
 #ifdef STM_NO_AUTOMATIC_SETJMP
     long repeat_count = did_abort;    /* test/support.py */
@@ -1160,6 +1158,9 @@
 #else
     long repeat_count = stm_rewind_jmp_setjmp(tl);
 #endif
+
+    start_timer();
+
     if (repeat_count) {
         /* only if there was an abort, we need to reset the memory: */
         if (tl->mem_reset_on_abort)
@@ -1167,7 +1168,7 @@
                    tl->mem_bytes_to_reset_on_abort);
     }
 
-    // _do_start_transaction is instrumented as well and pauses for waits
+    // _do_start_transaction is instrumented as well b/c it needs to pause for waits
     pause_timer();
     _do_start_transaction(tl);
     continue_timer();
diff --git a/c8/stm/timing.h b/c8/stm/timing.h
--- a/c8/stm/timing.h
+++ b/c8/stm/timing.h
@@ -29,7 +29,7 @@
 
 #define stm_duration_payload(duration_data)                                 \
     stm_timing_event_payload_data_t stm_duration_data =                     \
-        { .duration = &(duration_data) };                                     \
+        { .duration = &(duration_data) };                                   \
     stm_timing_event_payload_t stm_duration_payload =                       \
         { STM_EVENT_PAYLOAD_DURATION, stm_duration_data };
 

From pypy.commits at gmail.com  Sun Jun 11 17:22:41 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 14:22:41 -0700 (PDT)
Subject: [pypy-commit] pypy default: Support for the Python3 version of
 os.stat()
Message-ID: <593db4a1.94da1c0a.356e8.0386@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91577:6f7b1081c651
Date: 2017-06-11 23:11 +0200
http://bitbucket.org/pypy/pypy/changeset/6f7b1081c651/

Log:	Support for the Python3 version of os.stat()

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -600,6 +600,28 @@
         path = traits.as_str0(path)
         return win32_xstat(traits, path, traverse=False)
 
+ at specialize.argtype(0)
+def stat3(path):
+    # On Windows, the algorithm behind os.stat() changed a lot between
+    # Python 2 and Python 3.  This is the Python 3 version.
+    if not _WIN32:
+        return stat(path)
+    else:
+        traits = _preferred_traits(path)
+        path = traits.as_str0(path)
+        return win32_xstat3(traits, path, traverse=True)
+
+ at specialize.argtype(0)
+def lstat3(path):
+    # On Windows, the algorithm behind os.lstat() changed a lot between
+    # Python 2 and Python 3.  This is the Python 3 version.
+    if not _WIN32:
+        return lstat(path)
+    else:
+        traits = _preferred_traits(path)
+        path = traits.as_str0(path)
+        return win32_xstat3(traits, path, traverse=False)
+
 if rposix.HAVE_FSTATAT:
     from rpython.rlib.rposix import AT_FDCWD, AT_SYMLINK_NOFOLLOW
     c_fstatat = rffi.llexternal('fstatat64' if _LINUX else 'fstatat',
@@ -658,6 +680,45 @@
             return win32_attribute_data_to_stat(win32traits, data)
 
     @specialize.arg(0)
+    def win32_xstat3(traits, path, traverse=False):
+        # This is the Python3 version of os.stat() or lstat().
+        # XXX 'traverse' is ignored, and everything related to
+        # the "reparse points" is missing
+        win32traits = make_win32_traits(traits)
+
+        hFile = win32traits.CreateFile(path,
+            win32traits.FILE_READ_ATTRIBUTES,
+            0,
+            lltype.nullptr(rwin32.LPSECURITY_ATTRIBUTES.TO),
+            win32traits.OPEN_EXISTING,
+            win32traits.FILE_ATTRIBUTE_NORMAL |
+                win32traits.FILE_FLAG_BACKUP_SEMANTICS |
+                0,  # win32traits.FILE_FLAG_OPEN_REPARSE_POINT,
+            rwin32.NULL_HANDLE)
+
+        if hFile == rwin32.INVALID_HANDLE_VALUE:
+            errcode = rwin32.GetLastError_saved()
+            if (errcode != win32traits.ERROR_ACCESS_DENIED and
+                errcode != win32traits.ERROR_SHARING_VIOLATION):
+                raise WindowsError(errcode, "os_stat failed")
+
+            with lltype.scoped_alloc(
+                    win32traits.WIN32_FILE_ATTRIBUTE_DATA) as data:
+                if win32_attributes_from_dir(win32traits, path, data) == 0:
+                    raise WindowsError(rwin32.GetLastError_saved(),
+                                       "win32_attributes_from_dir failed")
+                return win32_attribute_data_to_stat(win32traits, data)
+
+        with lltype.scoped_alloc(
+                win32traits.BY_HANDLE_FILE_INFORMATION) as data:
+            res = rwin32.GetFileInformationByHandle(hFile, data)
+            errcode = rwin32.GetLastError_saved()
+            win32.CloseHandle(hFile)
+            if res == 0:
+                raise WindowsError(errcode, "GetFileInformationByHandle failed")
+            return win32_by_handle_info_to_stat(win32traits, data)
+
+    @specialize.arg(0)
     def win32_attributes_to_mode(win32traits, attributes):
         m = 0
         attributes = intmask(attributes)
@@ -697,10 +758,11 @@
 
         # specific to fstat()
         st_ino = make_longlong(info.c_nFileIndexHigh, info.c_nFileIndexLow)
+        st_dev = info.c_dwVolumeSerialNumber
         st_nlink = info.c_nNumberOfLinks
 
         result = (st_mode,
-                  st_ino, 0, st_nlink, 0, 0,
+                  st_ino, st_dev, st_nlink, 0, 0,
                   st_size,
                   atime, mtime, ctime,
                   extra_atime, extra_mtime, extra_ctime)
@@ -724,4 +786,3 @@
             return 1
         finally:
             lltype.free(filedata, flavor='raw')
-
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -46,6 +46,8 @@
             'INVALID_FILE_ATTRIBUTES')
         ERROR_SHARING_VIOLATION = platform.ConstantInteger(
             'ERROR_SHARING_VIOLATION')
+        ERROR_ACCESS_DENIED = platform.ConstantInteger(
+            'ERROR_ACCESS_DENIED')
         MOVEFILE_REPLACE_EXISTING = platform.ConstantInteger(
             'MOVEFILE_REPLACE_EXISTING')
         _S_IFDIR = platform.ConstantInteger('_S_IFDIR')
@@ -56,10 +58,14 @@
         FILE_TYPE_CHAR = platform.ConstantInteger('FILE_TYPE_CHAR')
         FILE_TYPE_PIPE = platform.ConstantInteger('FILE_TYPE_PIPE')
 
+        FILE_READ_ATTRIBUTES = platform.ConstantInteger(
+            'FILE_READ_ATTRIBUTES')
         FILE_WRITE_ATTRIBUTES = platform.ConstantInteger(
             'FILE_WRITE_ATTRIBUTES')
         OPEN_EXISTING = platform.ConstantInteger(
             'OPEN_EXISTING')
+        FILE_ATTRIBUTE_NORMAL = platform.ConstantInteger(
+            'FILE_ATTRIBUTE_NORMAL')
         FILE_FLAG_BACKUP_SEMANTICS = platform.ConstantInteger(
             'FILE_FLAG_BACKUP_SEMANTICS')
         VOLUME_NAME_DOS = platform.ConstantInteger('VOLUME_NAME_DOS')
@@ -103,10 +109,12 @@
                        INVALID_FILE_ATTRIBUTES
                        _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO
                        FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE
+                       FILE_READ_ATTRIBUTES
                        FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
                        VOLUME_NAME_DOS VOLUME_NAME_NT
                        ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
                        ERROR_SHARING_VIOLATION MOVEFILE_REPLACE_EXISTING
+                       ERROR_ACCESS_DENIED
                     '''.split():
             locals()[name] = config[name]
         LPWIN32_FIND_DATA    = lltype.Ptr(WIN32_FIND_DATA)
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -68,6 +68,16 @@
             py.test.skip("the underlying os.fstatvfs() failed: %s" % e)
         rposix_stat.fstatvfs(0)
 
+    @py.test.mark.skipif(sys.platform != 'win32', reason='win32 test')
+    def test_stat3_ino_dev(self):
+        st = rposix.stat2('C:\\')
+        assert st.st_dev == st.st_ino == 0
+        st = rposix.stat3('C:\\')
+        assert st.st_dev != 0 and st.st_ino != 0
+        st2 = rposix.lstat3('C:\\')
+        assert (st2.st_dev, st2.st_ino) == (st.st_dev, st.st_ino)
+
+
 @py.test.mark.skipif("not hasattr(rposix_stat, 'fstatat')")
 def test_fstatat(tmpdir):
     tmpdir.join('file').write('text')

From pypy.commits at gmail.com  Sun Jun 11 17:22:45 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 14:22:45 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <593db4a5.8bae1c0a.2bda1.dd68@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91579:6e54776fdedf
Date: 2017-06-11 23:21 +0200
http://bitbucket.org/pypy/pypy/changeset/6e54776fdedf/

Log:	hg merge default

diff too long, truncating to 2000 out of 4581 lines

diff --git a/.hgtags b/.hgtags
--- a/.hgtags
+++ b/.hgtags
@@ -38,3 +38,5 @@
 b16a4363e930f6401bceb499b9520955504c6cb0 release-pypy3.5-v5.7.0
 1aa2d8e03cdfab54b7121e93fda7e98ea88a30bf release-pypy2.7-v5.7.1
 2875f328eae2216a87f3d6f335092832eb031f56 release-pypy3.5-v5.7.1
+c925e73810367cd960a32592dd7f728f436c125c release-pypy2.7-v5.8.0
+a37ecfe5f142bc971a86d17305cc5d1d70abec64 release-pypy3.5-v5.8.0
diff --git a/LICENSE b/LICENSE
--- a/LICENSE
+++ b/LICENSE
@@ -39,11 +39,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -101,28 +101,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -149,13 +149,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -164,11 +164,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -183,6 +183,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -220,12 +221,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -264,7 +267,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -303,6 +305,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -340,11 +343,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py
--- a/lib-python/2.7/warnings.py
+++ b/lib-python/2.7/warnings.py
@@ -309,9 +309,12 @@
 
     def __init__(self, message, category, filename, lineno, file=None,
                     line=None):
-        local_values = locals()
-        for attr in self._WARNING_DETAILS:
-            setattr(self, attr, local_values[attr])
+        self.message = message
+        self.category = category
+        self.filename = filename
+        self.lineno = lineno
+        self.file = file
+        self.line = line
         self._category_name = category.__name__ if category else None
 
     def __str__(self):
diff --git a/lib_pypy/_ctypes/array.py b/lib_pypy/_ctypes/array.py
--- a/lib_pypy/_ctypes/array.py
+++ b/lib_pypy/_ctypes/array.py
@@ -74,12 +74,16 @@
         return self._type_._alignmentofinstances()
 
     def _CData_output(self, resarray, base=None, index=-1):
-        # this seems to be a string if we're array of char, surprise!
-        from ctypes import c_char, c_wchar
-        if self._type_ is c_char:
-            return _rawffi.charp2string(resarray.buffer, self._length_)
-        if self._type_ is c_wchar:
-            return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
+        from _rawffi.alt import types
+        # If a char_p or unichar_p is received, skip the string interpretation
+        if base._ffiargtype != types.Pointer(types.char_p) and \
+           base._ffiargtype != types.Pointer(types.unichar_p):
+            # this seems to be a string if we're array of char, surprise!
+            from ctypes import c_char, c_wchar
+            if self._type_ is c_char:
+                return _rawffi.charp2string(resarray.buffer, self._length_)
+            if self._type_ is c_wchar:
+                return _rawffi.wcharp2unicode(resarray.buffer, self._length_)
         res = self.__new__(self)
         ffiarray = self._ffiarray.fromaddress(resarray.buffer, self._length_)
         res._buffer = ffiarray
diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h
--- a/lib_pypy/cffi/_cffi_include.h
+++ b/lib_pypy/cffi/_cffi_include.h
@@ -159,9 +159,9 @@
 #define _cffi_from_c_struct                                              \
     ((PyObject *(*)(char *, struct _cffi_ctypedescr *))_cffi_exports[18])
 #define _cffi_to_c_wchar_t                                               \
-    ((wchar_t(*)(PyObject *))_cffi_exports[19])
+    ((_cffi_wchar_t(*)(PyObject *))_cffi_exports[19])
 #define _cffi_from_c_wchar_t                                             \
-    ((PyObject *(*)(wchar_t))_cffi_exports[20])
+    ((PyObject *(*)(_cffi_wchar_t))_cffi_exports[20])
 #define _cffi_to_c_long_double                                           \
     ((long double(*)(PyObject *))_cffi_exports[21])
 #define _cffi_to_c__Bool                                                 \
@@ -174,7 +174,11 @@
 #define _CFFI_CPIDX  25
 #define _cffi_call_python                                                \
     ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
-#define _CFFI_NUM_EXPORTS 26
+#define _cffi_to_c_wchar3216_t                                           \
+    ((int(*)(PyObject *))_cffi_exports[26])
+#define _cffi_from_c_wchar3216_t                                         \
+    ((PyObject *(*)(int))_cffi_exports[27])
+#define _CFFI_NUM_EXPORTS 28
 
 struct _cffi_ctypedescr;
 
@@ -215,6 +219,46 @@
     return NULL;
 }
 
+
+#ifdef HAVE_WCHAR_H
+typedef wchar_t _cffi_wchar_t;
+#else
+typedef uint16_t _cffi_wchar_t;   /* same random pick as _cffi_backend.c */
+#endif
+
+_CFFI_UNUSED_FN static uint16_t _cffi_to_c_char16_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return (uint16_t)_cffi_to_c_wchar_t(o);
+    else
+        return (uint16_t)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char16_t(uint16_t x)
+{
+    if (sizeof(_cffi_wchar_t) == 2)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+_CFFI_UNUSED_FN static int _cffi_to_c_char32_t(PyObject *o)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return (int)_cffi_to_c_wchar_t(o);
+    else
+        return (int)_cffi_to_c_wchar3216_t(o);
+}
+
+_CFFI_UNUSED_FN static PyObject *_cffi_from_c_char32_t(int x)
+{
+    if (sizeof(_cffi_wchar_t) == 4)
+        return _cffi_from_c_wchar_t(x);
+    else
+        return _cffi_from_c_wchar3216_t(x);
+}
+
+
 /**********  end CPython-specific section  **********/
 #else
 _CFFI_UNUSED_FN
diff --git a/lib_pypy/cffi/cffi_opcode.py b/lib_pypy/cffi/cffi_opcode.py
--- a/lib_pypy/cffi/cffi_opcode.py
+++ b/lib_pypy/cffi/cffi_opcode.py
@@ -107,9 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -135,6 +136,8 @@
     'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -122,6 +122,8 @@
         '_Bool':              'i',
         # the following types are not primitive in the C sense
         'wchar_t':            'c',
+        'char16_t':           'c',
+        'char32_t':           'c',
         'int8_t':             'i',
         'uint8_t':            'i',
         'int16_t':            'i',
diff --git a/lib_pypy/cffi/parse_c_type.h b/lib_pypy/cffi/parse_c_type.h
--- a/lib_pypy/cffi/parse_c_type.h
+++ b/lib_pypy/cffi/parse_c_type.h
@@ -81,8 +81,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -3,8 +3,9 @@
 from .error import VerificationError
 from .cffi_opcode import *
 
-VERSION = "0x2601"
-VERSION_EMBEDDED = "0x2701"
+VERSION_BASE = 0x2601
+VERSION_EMBEDDED = 0x2701
+VERSION_CHAR16CHAR32 = 0x2801
 
 
 class GlobalExpr:
@@ -126,6 +127,10 @@
         self.ffi = ffi
         self.module_name = module_name
         self.target_is_python = target_is_python
+        self._version = VERSION_BASE
+
+    def needs_version(self, ver):
+        self._version = max(self._version, ver)
 
     def collect_type_table(self):
         self._typesdict = {}
@@ -304,9 +309,7 @@
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
             prnt(''.join(lines))
-            version = VERSION_EMBEDDED
-        else:
-            version = VERSION
+            self.needs_version(VERSION_EMBEDDED)
         #
         # then paste the C source given by the user, verbatim.
         prnt('/************************************************************/')
@@ -405,7 +408,7 @@
             prnt('        _cffi_call_python_org = '
                  '(void(*)(struct _cffi_externpy_s *, char *))p[1];')
             prnt('    }')
-        prnt('    p[0] = (const void *)%s;' % version)
+        prnt('    p[0] = (const void *)0x%x;' % self._version)
         prnt('    p[1] = &_cffi_type_context;')
         prnt('}')
         # on Windows, distutils insists on putting init_cffi_xyz in
@@ -423,21 +426,22 @@
         prnt('PyMODINIT_FUNC')
         prnt('PyInit_%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  return _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  return _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#else')
         prnt('PyMODINIT_FUNC')
         prnt('init%s(void)' % (base_module_name,))
         prnt('{')
-        prnt('  _cffi_init("%s", %s, &_cffi_type_context);' % (
-            self.module_name, version))
+        prnt('  _cffi_init("%s", 0x%x, &_cffi_type_context);' % (
+            self.module_name, self._version))
         prnt('}')
         prnt('#endif')
         prnt()
         prnt('#ifdef __GNUC__')
         prnt('#  pragma GCC visibility pop')
         prnt('#endif')
+        self._version = None
 
     def _to_py(self, x):
         if isinstance(x, str):
@@ -476,7 +480,8 @@
             prnt('from %s import ffi as _ffi%d' % (included_module_name, i))
         prnt()
         prnt("ffi = _cffi_backend.FFI('%s'," % (self.module_name,))
-        prnt("    _version = %s," % (VERSION,))
+        prnt("    _version = 0x%x," % (self._version,))
+        self._version = None
         #
         # the '_types' keyword argument
         self.cffi_types = tuple(self.cffi_types)    # don't change any more
@@ -515,8 +520,11 @@
                 # double' here, and _cffi_to_c_double would loose precision
                 converter = '(%s)_cffi_to_c_double' % (tp.get_c_name(''),)
             else:
-                converter = '(%s)_cffi_to_c_%s' % (tp.get_c_name(''),
+                cname = tp.get_c_name('')
+                converter = '(%s)_cffi_to_c_%s' % (cname,
                                                    tp.name.replace(' ', '_'))
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
             errvalue = '-1'
         #
         elif isinstance(tp, model.PointerType):
@@ -573,7 +581,10 @@
             elif isinstance(tp, model.UnknownFloatType):
                 return '_cffi_from_c_double(%s)' % (var,)
             elif tp.name != 'long double' and not tp.is_complex_type():
-                return '_cffi_from_c_%s(%s)' % (tp.name.replace(' ', '_'), var)
+                cname = tp.name.replace(' ', '_')
+                if cname in ('char16_t', 'char32_t'):
+                    self.needs_version(VERSION_CHAR16CHAR32)
+                return '_cffi_from_c_%s(%s)' % (cname, var)
             else:
                 return '_cffi_from_c_deref((char *)&%s, _cffi_type(%d))' % (
                     var, self._gettypenum(tp))
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -808,7 +808,8 @@
 #include <stddef.h>
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -842,11 +843,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -627,7 +627,8 @@
 #include <sys/types.h>   /* XXX for ssize_t on some platforms */
 
 /* this block of #ifs should be kept exactly identical between
-   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py
+   and cffi/_cffi_include.h */
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 # if _MSC_VER < 1600   /* MSVC < 2010 */
@@ -661,11 +662,13 @@
 #  include <stdint.h>
 # endif
 # if _MSC_VER < 1800   /* MSVC < 2013 */
-   typedef unsigned char _Bool;
+#  ifndef __cplusplus
+    typedef unsigned char _Bool;
+#  endif
 # endif
 #else
 # include <stdint.h>
-# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux)
 #  include <alloca.h>
 # endif
 #endif
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -195,6 +195,29 @@
 ``/tmp/usession-YOURNAME/build/``.  You can then either move the file
 hierarchy or unpack the ``.tar.bz2`` at the correct place.
 
+It is recommended to use package.py because custom scripts will
+invariably become out-of-date.  If you want to write custom scripts
+anyway, note an easy-to-miss point: some modules are written with CFFI,
+and require some compilation.  If you install PyPy as root without
+pre-compiling them, normal users will get errors:
+
+* PyPy 2.5.1 or earlier: normal users would see permission errors.
+  Installers need to run ``pypy -c "import gdbm"`` and other similar
+  commands at install time; the exact list is in `package.py`_.  Users
+  seeing a broken installation of PyPy can fix it after-the-fact if they
+  have sudo rights, by running once e.g. ``sudo pypy -c "import gdbm``.
+
+* PyPy 2.6 and later: anyone would get ``ImportError: no module named
+  _gdbm_cffi``.  Installers need to run ``pypy _gdbm_build.py`` in the
+  ``lib_pypy`` directory during the installation process (plus others;
+  see the exact list in `package.py`_).  Users seeing a broken
+  installation of PyPy can fix it after-the-fact, by running ``pypy
+  /path/to/lib_pypy/_gdbm_build.py``.  This command produces a file
+  called ``_gdbm_cffi.pypy-41.so`` locally, which is a C extension
+  module for PyPy.  You can move it at any place where modules are
+  normally found: e.g. in your project's main directory, or in a
+  directory that you add to the env var ``PYTHONPATH``.
+
 
 Installation
 ------------
diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py
--- a/pypy/doc/conf.py
+++ b/pypy/doc/conf.py
@@ -59,16 +59,16 @@
 
 # General information about the project.
 project = u'PyPy'
-copyright = u'2016, The PyPy Project'
+copyright = u'2017, The PyPy Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '5.4'
+version = '5.8'
 # The full version, including alpha/beta/rc tags.
-release = '5.4.0'
+release = '5.8.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst
--- a/pypy/doc/contributor.rst
+++ b/pypy/doc/contributor.rst
@@ -6,11 +6,11 @@
 
   Armin Rigo
   Maciej Fijalkowski
-  Carl Friedrich Bolz
+  Carl Friedrich Bolz-Tereick
   Amaury Forgeot d'Arc
   Antonio Cuni
+  Matti Picus
   Samuele Pedroni
-  Matti Picus
   Ronan Lamy
   Alex Gaynor
   Philip Jenvey
@@ -68,28 +68,28 @@
   Vincent Legoll
   Michael Foord
   Stephan Diehl
+  Stefano Rivera
   Stefan Schwarzer
   Tomek Meka
   Valentino Volonghi
-  Stefano Rivera
   Patrick Maupin
   Devin Jeanpierre
   Bob Ippolito
   Bruno Gola
   David Malcolm
   Jean-Paul Calderone
+  Squeaky
   Edd Barrett
-  Squeaky
   Timo Paulssen
   Marius Gedminas
   Alexandre Fayolle
   Simon Burton
   Nicolas Truessel
   Martin Matusiak
+  Laurence Tratt
   Wenzhu Man
   Konstantin Lopuhin
   John Witulski
-  Laurence Tratt
   Greg Price
   Ivan Sichmann Freitas
   Dario Bertini
@@ -116,13 +116,13 @@
   Stian Andreassen
   Wanja Saatkamp
   Mike Blume
+  Joannah Nanjekye
   Gerald Klix
   Oscar Nierstrasz
   Rami Chowdhury
   Stefan H. Muller
-  Joannah Nanjekye
+  Tim Felgentreff
   Eugene Oden
-  Tim Felgentreff
   Jeff Terrace
   Henry Mason
   Vasily Kuznetsov
@@ -131,11 +131,11 @@
   Dusty Phillips
   Lukas Renggli
   Guenter Jantzen
+  Jasper Schulz
   Ned Batchelder
   Amit Regmi
   Anton Gulenko
   Sergey Matyunin
-  Jasper Schulz
   Andrew Chambers
   Nicolas Chauvat
   Andrew Durdin
@@ -150,6 +150,7 @@
   Gintautas Miliauskas
   Lucian Branescu Mihaila
   anatoly techtonik
+  Dodan Mihai
   Karl Bartel
   Gabriel Lavoie
   Jared Grubb
@@ -187,12 +188,14 @@
   Vaibhav Sood
   Reuben Cummings
   Attila Gobi
+  Alecsandru Patrascu
   Christopher Pope
   Tristan Arthur
   Christian Tismer 
   Dan Stromberg
   Carl Meyer
   Florin Papa
+  Jens-Uwe Mager
   Valentina Mukhamedzhanova
   Stefano Parmesan
   touilleMan
@@ -231,7 +234,6 @@
   Dan Buch
   Lene Wagner
   Tomo Cocoa
-  Alecsandru Patrascu
   David Lievens
   Neil Blakey-Milner
   Henrik Vendelbo
@@ -270,6 +272,7 @@
   Anna Katrina Dominguez
   Kim Jin Su
   Amber Brown
+  Nate Bragg
   Ben Darnell
   Juan Francisco Cantero Hurtado
   Godefroid Chappelle
@@ -307,11 +310,13 @@
   Jim Hunziker
   shoma hosaka
   Buck Golemon
+  Iraklis D.
   JohnDoe
   yrttyr
   Michael Chermside
   Anna Ravencroft
   remarkablerocket
+  Petre Vijiac
   Berker Peksag
   Christian Muirhead
   soareschen
diff --git a/pypy/doc/discussion/finalizer-order.rst b/pypy/doc/discussion/finalizer-order.rst
--- a/pypy/doc/discussion/finalizer-order.rst
+++ b/pypy/doc/discussion/finalizer-order.rst
@@ -60,7 +60,7 @@
 The interface for full finalizers is made with PyPy in mind, but should
 be generally useful.
 
-The idea is that you subclass the ``rgc.FinalizerQueue`` class::
+The idea is that you subclass the ``rgc.FinalizerQueue`` class:
 
 * You must give a class-level attribute ``base_class``, which is the
   base class of all instances with a finalizer.  (If you need
diff --git a/pypy/doc/discussion/rawrefcount.rst b/pypy/doc/discussion/rawrefcount.rst
--- a/pypy/doc/discussion/rawrefcount.rst
+++ b/pypy/doc/discussion/rawrefcount.rst
@@ -68,10 +68,12 @@
 and O = list of links created with rawrefcount.create_link_pyobj().
 The PyPy objects in the list O are all W_CPyExtPlaceHolderObject: all
 the data is in the PyObjects, and all outsite references (if any) are
-in C, as "PyObject *" fields.
+in C, as ``PyObject *`` fields.
 
 So, during the collection we do this about P links:
 
+.. code-block:: python
+
     for (p, ob) in P:
         if ob->ob_refcnt != REFCNT_FROM_PYPY
                and ob->ob_refcnt != REFCNT_FROM_PYPY_LIGHT:
@@ -80,6 +82,8 @@
 At the end of the collection, the P and O links are both handled like
 this:
 
+.. code-block:: python
+
     for (p, ob) in P + O:
         if p is not surviving:    # even if 'ob' might be surviving
             unlink p and ob
diff --git a/pypy/doc/how-to-release.rst b/pypy/doc/how-to-release.rst
--- a/pypy/doc/how-to-release.rst
+++ b/pypy/doc/how-to-release.rst
@@ -5,8 +5,8 @@
 ++++++++++++++
 
 We try to create a stable release a few times a year. These are released on
-a branch named like release-2.x or release-4.x, and each release is tagged,
-for instance release-4.0.1. 
+a branch named like release-pypy3.5-v2.x or release-pypy3.5-v4.x, and each
+release is tagged, for instance release-pypy3.5-v4.0.1. 
 
 After release, inevitably there are bug fixes. It is the responsibility of
 the commiter who fixes a bug to make sure this fix is on the release branch,
@@ -33,7 +33,7 @@
 * If needed, make a release branch
 * Bump the
   pypy version number in module/sys/version.py and in
-  module/cpyext/include/patchlevel.h and . The branch
+  module/cpyext/include/patchlevel.h and in doc/conf.py. The branch
   will capture the revision number of this change for the release.
 
   Some of the next updates may be done before or after branching; make
diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst
--- a/pypy/doc/index-of-whatsnew.rst
+++ b/pypy/doc/index-of-whatsnew.rst
@@ -30,12 +30,22 @@
    whatsnew-2.0.0-beta1.rst
    whatsnew-1.9.rst
 
+CPython 3.5 compatible versions
+-------------------------------
+
+.. toctree::
+
+   whatsnew-pypy3-head.rst
+   whatsnew-pypy3-5.8.0.rst
+   whatsnew-pypy3-5.7.0.rst
+
 CPython 3.3 compatible versions
 -------------------------------
 
 .. toctree::
 
    whatsnew-pypy3-5.5.0.rst
+   whatsnew-pypy3-5.1.1-alpha1.rst
 
 CPython 3.2 compatible versions
 -------------------------------
diff --git a/pypy/doc/install.rst b/pypy/doc/install.rst
--- a/pypy/doc/install.rst
+++ b/pypy/doc/install.rst
@@ -12,6 +12,7 @@
 and using pip. 
 
 .. _prebuilt-pypy:
+
 Download a pre-built PyPy
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/pypy/doc/objspace.rst b/pypy/doc/objspace.rst
--- a/pypy/doc/objspace.rst
+++ b/pypy/doc/objspace.rst
@@ -250,12 +250,12 @@
 .. py:function:: newunicode(ustr)
 
    Creates a Unicode string from an rpython unicode string.
-   This method may disappear soon and be replaced by :py:function:`newutf8()`.
+   This method may disappear soon and be replaced by :py:function::`newutf8`.
 
 .. py:function:: newutf8(bytestr)
 
    Creates a Unicode string from an rpython byte string, decoded as
-   "utf-8-nosg".  On PyPy3 it is the same as :py:function:`newtext()`.
+   "utf-8-nosg".  On PyPy3 it is the same as :py:function::`newtext`.
 
 Many more space operations can be found in `pypy/interpeter/baseobjspace.py` and
 `pypy/objspace/std/objspace.py`.
@@ -302,9 +302,9 @@
 
 .. py:function:: unicode_w(w_x)
 
-   Takes an application level :py:class:`unicode` and return an
+   Takes an application level :py:class::`unicode` and return an
    interpreter-level unicode string.  This method may disappear soon and
-   be replaced by :py:function:`text_w()`.
+   be replaced by :py:function::`text_w`.
 
 .. py:function:: float_w(w_x)
 
diff --git a/pypy/doc/release-pypy2.7-v5.4.0.rst b/pypy/doc/release-pypy2.7-v5.4.0.rst
--- a/pypy/doc/release-pypy2.7-v5.4.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.4.0.rst
@@ -171,7 +171,7 @@
 * Performance improvements:
 
   * Add a before_call()-like equivalent before a few operations like
-   `malloc_nursery`, to move values from registers into other registers
+    `malloc_nursery`, to move values from registers into other registers
     instead of to the stack.
 
   * More tightly pack the stack when calling with `release gil`
diff --git a/pypy/doc/release-pypy2.7-v5.6.0.rst b/pypy/doc/release-pypy2.7-v5.6.0.rst
--- a/pypy/doc/release-pypy2.7-v5.6.0.rst
+++ b/pypy/doc/release-pypy2.7-v5.6.0.rst
@@ -140,7 +140,7 @@
     preamble
   * In JIT residual calls, if the called function starts with a fast-path like
     ``if x.foo != 0: return x.foo``, then inline the check before doing the
-     ``CALL``.
+    ``CALL``.
   * Ensure ``make_inputargs`` fails properly when given arguments with type 
     information
   * Makes ``optimiseopt`` iterative instead of recursive so it can be reasoned
diff --git a/pypy/doc/release-v5.8.0.rst b/pypy/doc/release-v5.8.0.rst
--- a/pypy/doc/release-v5.8.0.rst
+++ b/pypy/doc/release-v5.8.0.rst
@@ -3,33 +3,44 @@
 =====================================
 
 The PyPy team is proud to release both PyPy2.7 v5.8 (an interpreter supporting
-Python v2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
-v3.5 syntax). The two releases are both based on much the same codebase, thus
+Python 2.7 syntax), and a beta-quality PyPy3.5 v5.8 (an interpreter for Python
+3.5 syntax). The two releases are both based on much the same codebase, thus
 the dual release.  Note that PyPy3.5 supports Linux 64bit only for now. 
 
 This new PyPy2.7 release includes the upstream stdlib version 2.7.13, and
 PyPy3.5 includes the upstream stdlib version 3.5.3.
 
-This release enables `profile guided optimization` of the base interpreter,
-which may make unjitted code run faster.
+We fixed critical bugs in the shadowstack_ rootfinder garbage collector
+strategy that crashed multithreaded programs and very rarely showed up
+even in single threaded programs.
+
+We added native PyPy support to profile frames in the vmprof_ statistical
+profiler.
+
+The ``struct`` module functions ``pack*`` and ``unpack*`` are now much faster,
+especially on raw buffers and bytearrays. Microbenchmarks show a 2x to 10x
+speedup. Thanks to `Gambit Research`_ for sponsoring this work.
+
+This release adds (but disables by default) link-time optimization and
+`profile guided optimization`_ of the base interpreter, which may make
+unjitted code run faster. To use these, translate with appropriate
+`options`_.  Be aware of `issues with gcc toolchains`_, though.
 
 Please let us know if your use case is slow, we have ideas how to make things
 faster but need real-world examples (not micro-benchmarks) of problematic code.
 
-Work proceeds at a good pace on the PyPy3.5
-version due to a grant_ from the Mozilla Foundation, hence our 3.5.3 beta
-release. Thanks Mozilla !!! While we do not pass all tests yet, asyncio works and
-as `these benchmarks show`_ it already gives a nice speed bump.
-We also backported the ``f""`` formatting from 3.6 (as an exception; otherwise
-"PyPy3.5" supports the Python 3.5 language).
+Work sponsored by a Mozilla grant_ continues on PyPy3.5; numerous fixes from
+CPython were ported to PyPy and PEP 489 was fully implemented. Of course the
+bug fixes and performance enhancements mentioned above are part of both PyPy
+2.7 and PyPy 3.5.
 
 CFFI_, which is part of the PyPy release, has been updated to an unreleased 1.10.1,
 improving an already great package for interfacing with C.
 
-As always, this release fixed many issues and bugs raised by the
+As always, this release fixed many other issues and bugs raised by the
 growing community of PyPy users. We strongly recommend updating.
 
-You can download the v5.8 release here:
+You can download the v5.8 releases here:
 
     http://pypy.org/download.html
 
@@ -43,13 +54,18 @@
 with making RPython's JIT even better.
 
 .. _`profile guided optimization`: https://pythonfiles.wordpress.com/2017/05/12/enabling-profile-guided-optimizations-for-pypy
+.. _shadowstack: config/translation.gcrootfinder.html
+.. _vmprof: http://vmprof.readthedocs.io
+.. _`issues with gcc toolchains`: https://bitbucket.org/pypy/pypy/issues/2572/link-time-optimization-lto-disabled
 .. _CFFI: https://cffi.readthedocs.io/en/latest/whatsnew.html
 .. _grant: https://morepypy.blogspot.com/2016/08/pypy-gets-funding-from-mozilla-for.html
 .. _`PyPy`: index.html
 .. _`RPython`: https://rpython.readthedocs.org
 .. _`modules`: project-ideas.html#make-more-python-modules-pypy-friendly
 .. _`help`: project-ideas.html
+.. _`options`: config/commandline.html#general-translation-options
 .. _`these benchmarks show`: https://morepypy.blogspot.com/2017/03/async-http-benchmarks-on-pypy3.html
+.. _`Gambit Research`: http://gambitresearch.com
 
 What is PyPy?
 =============
@@ -148,7 +164,7 @@
     accepted in a few more places, e.g. in compile()
 
 
-.. _here: http://rpython.readthedocs.io/en/latest/cpython_differences.html
+.. _here: cpython_differences.html
 
 Highlights of the PyPy3.5 release (since 5.7 beta released March 2017)
 ======================================================================
diff --git a/pypy/doc/tool/makecontributor.py b/pypy/doc/tool/makecontributor.py
--- a/pypy/doc/tool/makecontributor.py
+++ b/pypy/doc/tool/makecontributor.py
@@ -18,8 +18,9 @@
     'Antonio Cuni': ['antocuni', 'anto'],
     'Armin Rigo': ['arigo', 'arfigo', 'armin', 'arigato'],
     'Maciej Fijalkowski': ['fijal'],
-    'Carl Friedrich Bolz': ['cfbolz', 'cf'],
+    'Carl Friedrich Bolz-Tereick': ['Carl Friedrich Bolz', 'cfbolz', 'cf'],
     'Samuele Pedroni': ['pedronis', 'samuele', 'samule'],
+    'Richard Plangger':['planrich'],
     'Michael Hudson': ['mwh'],
     'Holger Krekel': ['hpk', 'holger krekel', 'holger', 'hufpk'],
     "Amaury Forgeot d'Arc": ['afa'],
@@ -67,7 +68,7 @@
     'Edd Barrett': ['edd'],
     'Manuel Jacob': ['mjacob'],
     'Rami Chowdhury': ['necaris'],
-    'Stanislaw Halik':['w31rd0'],
+    'Stanislaw Halik': ['Stanislaw Halik', 'w31rd0'],
     'Wenzhu Man':['wenzhu man', 'wenzhuman'],
     'Anton Gulenko':['anton gulenko', 'anton_gulenko'],
     'Richard Lancaster':['richardlancaster'],
@@ -78,7 +79,8 @@
     'Jasper Schulz':['Jasper.Schulz', 'jbs'],
     'Aaron Gallagher':['"Aaron Gallagher'],
     'Yasir Suhail':['yasirs'],
-    'Squeaky', ['squeaky'],
+    'Squeaky': ['squeaky'],
+    "Amaury Forgeot d'Arc": ['amauryfa at gmail.com'],
     }
 
 alias_map = {}
diff --git a/pypy/doc/whatsnew-2.6.1.rst b/pypy/doc/whatsnew-2.6.1.rst
--- a/pypy/doc/whatsnew-2.6.1.rst
+++ b/pypy/doc/whatsnew-2.6.1.rst
@@ -6,6 +6,7 @@
 .. startrev: 91904d5c5188
 
 .. branch: use_min_scalar
+
 Correctly resolve the output dtype of ufunc(array, scalar) calls.
 
 .. branch: stdlib-2.7.10
@@ -15,6 +16,7 @@
 .. branch: issue2062
 
 .. branch: disable-unroll-for-short-loops
+
 The JIT no longer performs loop unrolling if the loop compiles to too much code.
 
 .. branch: run-create_cffi_imports
@@ -32,9 +34,11 @@
 ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.)
 
 .. branch: cffi-callback-onerror
+
 Part of cffi 1.2.
 
 .. branch: cffi-new-allocator
+
 Part of cffi 1.2.
 
 .. branch: unicode-dtype
diff --git a/pypy/doc/whatsnew-4.0.0.rst b/pypy/doc/whatsnew-4.0.0.rst
--- a/pypy/doc/whatsnew-4.0.0.rst
+++ b/pypy/doc/whatsnew-4.0.0.rst
@@ -6,23 +6,28 @@
 .. startrev: 3a8f5481dab4
 
 .. branch: keys_with_hash
+
 Improve the performance of ``dict.update()`` and a bunch of methods from
 sets, by reusing the hash value stored in one dict when inspecting
 or changing another dict with that key.
 
 .. branch: optresult-unroll 
+
 A major refactoring of the ``ResOperations`` that kills Box. Also rewrote
 unrolling to enable future enhancements.  Should improve warmup time
 by 20% or so.
 
 .. branch: optimize-cond-call
+
 Optimize common sequences of operations like
 ``int_lt/cond_call`` in the JIT backends
 
 .. branch: missing_openssl_include
+
 Fix for missing headers in OpenBSD, already applied in downstream ports
 
 .. branch: gc-more-incremental
+
 Remove a source of non-incremental-ness in the GC: now
 ``external_malloc()`` no longer runs ``gc_step_until()`` any more. If there
 is a currently-running major collection, we do only so many steps
@@ -32,11 +37,13 @@
 keep adding up between them.
 
 .. branch: remember-tracing-counts
+
 Reenable jithooks
 
 .. branch: detect_egd2
 
 .. branch: shadowstack-no-move-2
+
 Issue #2141: fix a crash on Windows and OS/X and ARM when running
 at least 20 threads.
 
@@ -55,6 +62,7 @@
 floats, cf. issue #2148.
 
 .. branch: cffi-stdcall
+
 Win32: support ``__stdcall`` in CFFI.
 
 .. branch: callfamily
@@ -93,6 +101,7 @@
 .. branch: osx-libffi
 
 .. branch: lazy-fast2locals
-improve the performance of simple trace functions by lazily calling
+
+Improve the performance of simple trace functions by lazily calling
 ``fast2locals`` and ``locals2fast`` only if ``f_locals`` is actually accessed.
 
diff --git a/pypy/doc/whatsnew-5.0.0.rst b/pypy/doc/whatsnew-5.0.0.rst
--- a/pypy/doc/whatsnew-5.0.0.rst
+++ b/pypy/doc/whatsnew-5.0.0.rst
@@ -192,6 +192,7 @@
 Fix boolean-array indexing in micronumpy
 
 .. branch: numpy_partition
+
 Support ndarray.partition() as an app-level function numpy.core._partition_use,
 provided as a cffi wrapper to upstream's implementation in the pypy/numpy repo
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -6,5 +6,6 @@
 .. startrev: 558bd00b3dd8
 
 .. branch: cffi-complex
+.. branch: cffi-char16-char32
 
-Part of the upgrade to cffi 1.11
+The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
diff --git a/pypy/doc/whatsnew-pypy2-5.3.0.rst b/pypy/doc/whatsnew-pypy2-5.3.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.3.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.3.0.rst
@@ -29,6 +29,7 @@
 upstream numpy via cpyext, so we created (yet another) fork of numpy at 
 github.com/pypy/numpy with the needed changes. Among the significant changes 
 to cpyext:
+
   - allow c-snippet tests to be run with -A so we can verify we are compatible
   - fix many edge cases exposed by fixing tests to run with -A
   - issequence() logic matches cpython
diff --git a/pypy/doc/whatsnew-pypy2-5.4.0.rst b/pypy/doc/whatsnew-pypy2-5.4.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.4.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.4.0.rst
@@ -6,10 +6,12 @@
 .. startrev: 873218a739f1
 
 .. 418b05f95db5
+
 Improve CPython compatibility for ``is``. Now code like ``if x is ():``
 works the same way as it does on CPython.  See http://pypy.readthedocs.io/en/latest/cpython_differences.html#object-identity-of-primitive-values-is-and-id .
 
 .. pull request #455
+
 Add sys.{get,set}dlopenflags, for cpyext extensions.
 
 .. branch: fix-gen-dfa
@@ -36,9 +38,11 @@
 compatible.
 
 .. branch: pyfile-tell
+
 Sync w_file with the c-level FILE* before returning FILE* in PyFile_AsFile
 
 .. branch: rw-PyString_AS_STRING
+
 Allow rw access to the char* returned from PyString_AS_STRING, also refactor
 PyStringObject to look like cpython's and allow subclassing PyString_Type and
 PyUnicode_Type
diff --git a/pypy/doc/whatsnew-pypy2-5.6.0.rst b/pypy/doc/whatsnew-pypy2-5.6.0.rst
--- a/pypy/doc/whatsnew-pypy2-5.6.0.rst
+++ b/pypy/doc/whatsnew-pypy2-5.6.0.rst
@@ -12,18 +12,22 @@
 .. branch: testing-cleanup-py3k
 
 .. branch: rpython-resync
+
 Backport rpython changes made directly on the py3k and py3.5 branches.
 
 .. branch: buffer-interface
+
 Implement PyObject_GetBuffer, PyMemoryView_GET_BUFFER, and handles memoryviews
 in numpypy
 
 .. branch: force-virtual-state
+
 Improve merging of virtual states in the JIT in order to avoid jumping to the
 preamble. Accomplished by allocating virtual objects where non-virtuals are
 expected.
 
 .. branch: conditional_call_value_3
+
 JIT residual calls: if the called function starts with a fast-path
 like "if x.foo != 0: return x.foo", then inline the check before
 doing the CALL.  For now, string hashing is about the only case.
@@ -64,6 +68,7 @@
 
 
 .. fb6bb835369e
+
 Change the ``timeit`` module: it now prints the average time and the standard
 deviation over 7 runs by default, instead of the minimum. The minimum is often
 misleading.
@@ -75,9 +80,6 @@
 
 .. branch: Tiberiumk/fix-2412-1476011166874
 .. branch: redirect-assembler-jitlog
-
-
-
 .. branch: stdlib-2.7.12
 
 Update stdlib to version 2.7.12
diff --git a/pypy/doc/whatsnew-pypy3-5.8.0.rst b/pypy/doc/whatsnew-pypy3-5.8.0.rst
--- a/pypy/doc/whatsnew-pypy3-5.8.0.rst
+++ b/pypy/doc/whatsnew-pypy3-5.8.0.rst
@@ -6,6 +6,7 @@
 .. startrev: afbf09453369
 
 .. branch: mtest
+
 Use "<python> -m test" to run the CPython test suite, as documented by CPython,
 instead of our outdated regrverbose.py script.
 
diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py
--- a/pypy/module/_cffi_backend/cffi1_module.py
+++ b/pypy/module/_cffi_backend/cffi1_module.py
@@ -9,7 +9,7 @@
 
 
 VERSION_MIN    = 0x2601
-VERSION_MAX    = 0x27FF
+VERSION_MAX    = 0x28FF
 
 VERSION_EXPORT = 0x0A03
 
diff --git a/pypy/module/_cffi_backend/cffi_opcode.py b/pypy/module/_cffi_backend/cffi_opcode.py
--- a/pypy/module/_cffi_backend/cffi_opcode.py
+++ b/pypy/module/_cffi_backend/cffi_opcode.py
@@ -107,8 +107,10 @@
 PRIM_UINTMAX       = 47
 PRIM_FLOATCOMPLEX  = 48
 PRIM_DOUBLECOMPLEX = 49
+PRIM_CHAR16        = 50
+PRIM_CHAR32        = 51
 
-_NUM_PRIM          = 50
+_NUM_PRIM          = 52
 _UNKNOWN_PRIM          = -1
 _UNKNOWN_FLOAT_PRIM    = -2
 _UNKNOWN_LONG_DOUBLE   = -3
@@ -131,8 +133,12 @@
     'float':              PRIM_FLOAT,
     'double':             PRIM_DOUBLE,
     'long double':        PRIM_LONGDOUBLE,
+    'float _Complex':     PRIM_FLOATCOMPLEX,
+    'double _Complex':    PRIM_DOUBLECOMPLEX,
     '_Bool':              PRIM_BOOL,
     'wchar_t':            PRIM_WCHAR,
+    'char16_t':           PRIM_CHAR16,
+    'char32_t':           PRIM_CHAR32,
     'int8_t':             PRIM_INT8,
     'uint8_t':            PRIM_UINT8,
     'int16_t':            PRIM_INT16,
diff --git a/pypy/module/_cffi_backend/ctypearray.py b/pypy/module/_cffi_backend/ctypearray.py
--- a/pypy/module/_cffi_backend/ctypearray.py
+++ b/pypy/module/_cffi_backend/ctypearray.py
@@ -36,8 +36,7 @@
         datasize = self.size
         #
         if datasize < 0:
-            from pypy.module._cffi_backend import misc
-            w_init, length = misc.get_new_array_length(space, w_init)
+            w_init, length = self.get_new_array_length(w_init)
             try:
                 datasize = ovfcheck(length * self.ctitem.size)
             except OverflowError:
@@ -53,6 +52,29 @@
                 self.convert_from_object(ptr, w_init)
         return cdata
 
+    def get_new_array_length(self, w_value):
+        space = self.space
+        if (space.isinstance_w(w_value, space.w_list) or
+            space.isinstance_w(w_value, space.w_tuple)):
+            return (w_value, space.int_w(space.len(w_value)))
+        elif space.isinstance_w(w_value, space.w_bytes):
+            # from a string, we add the null terminator
+            s = space.bytes_w(w_value)
+            return (w_value, len(s) + 1)
+        elif space.isinstance_w(w_value, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_value)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            return (w_value, length + 1)
+        else:
+            explicitlength = space.getindex_w(w_value, space.w_OverflowError)
+            if explicitlength < 0:
+                raise oefmt(space.w_ValueError, "negative array length")
+            return (space.w_None, explicitlength)
+
     def _check_subscript_index(self, w_cdata, i):
         space = self.space
         if i < 0:
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -10,7 +10,7 @@
 from rpython.rtyper.tool import rfficache
 
 from pypy.interpreter.error import oefmt
-from pypy.module._cffi_backend import cdataobj, misc
+from pypy.module._cffi_backend import cdataobj, misc, wchar_helper
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
 
@@ -42,11 +42,13 @@
     def cast_unicode(self, w_ob):
         space = self.space
         s = space.unicode_w(w_ob)
-        if len(s) != 1:
+        try:
+            ordinal = wchar_helper.unicode_to_ordinal(s)
+        except ValueError:
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return intmask(ordinal)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
@@ -148,53 +150,83 @@
         return self.space.newbytes(s)
 
 
-# XXX explicitly use an integer type instead of lltype.UniChar here,
-# because for now the latter is defined as unsigned by RPython (even
-# though it may be signed when 'wchar_t' is written to C).
-WCHAR_INT = {(2, False): rffi.USHORT,
-             (4, False): rffi.UINT,
-             (4, True): rffi.INT}[rffi.sizeof(lltype.UniChar),
-                                  rfficache.signof_c_type('wchar_t')]
-WCHAR_INTP = rffi.CArrayPtr(WCHAR_INT)
+class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
+    _attrs_            = ['is_signed_wchar']
+    _immutable_fields_ = ['is_signed_wchar']
 
-class W_CTypePrimitiveUniChar(W_CTypePrimitiveCharOrUniChar):
-    _attrs_ = []
+    _wchar_is_signed = rfficache.signof_c_type('wchar_t')
 
-    if rffi.r_wchar_t.SIGN:
-        def write_raw_integer_data(self, w_cdata, value):
-            w_cdata.write_raw_signed_data(value)
+    def __init__(self, space, size, name, name_position, align):
+        W_CTypePrimitiveCharOrUniChar.__init__(self, space, size, name,
+                                               name_position, align)
+        self.is_signed_wchar = self._wchar_is_signed and (name == "wchar_t")
+        # "char16_t" and "char32_t" are always unsigned
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(WCHAR_INTP, cdata)
-        return self.space.newint(unichardata[0])
+        if self.is_signed_wchar:
+            value = misc.read_raw_long_data(cdata, self.size)
+            return self.space.newint(value)
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)
+            if self.size < rffi.sizeof(lltype.Signed):
+                return self.space.newint(intmask(value))
+            else:
+                return self.space.newint(value)    # r_uint => 'long' object
 
     def convert_to_object(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.newunicode(unichardata[0])
+        if self.is_signed_wchar:
+            unichardata = rffi.cast(rffi.CWCHARP, cdata)
+            return self.space.newunicode(unichardata[0])
+        else:
+            value = misc.read_raw_ulong_data(cdata, self.size)   # r_uint
+            try:
+                u = wchar_helper.ordinal_to_unicode(value)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
+            return self.space.newunicode(u)
 
     def string(self, cdataobj, maxlen):
         with cdataobj as ptr:
             w_res = self.convert_to_object(ptr)
         return w_res
 
-    def _convert_to_unichar(self, w_ob):
+    def _convert_to_charN_t(self, w_ob):
+        # returns a r_uint.  If self.size == 2, it is smaller than 0x10000
         space = self.space
         if space.isinstance_w(w_ob, space.w_unicode):
-            s = space.unicode_w(w_ob)
-            if len(s) == 1:
-                return s[0]
-        if (isinstance(w_ob, cdataobj.W_CData) and
-               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar)):
+            u = space.unicode_w(w_ob)
+            try:
+                ordinal = wchar_helper.unicode_to_ordinal(u)
+            except ValueError:
+                pass
+            else:
+                if self.size == 2 and ordinal > 0xffff:
+                    raise self._convert_error("single character <= 0xFFFF",
+                                              w_ob)
+                return ordinal
+        elif (isinstance(w_ob, cdataobj.W_CData) and
+               isinstance(w_ob.ctype, W_CTypePrimitiveUniChar) and
+               w_ob.ctype.size == self.size):
             with w_ob as ptr:
-                return rffi.cast(rffi.CWCHARP, ptr)[0]
+                return misc.read_raw_ulong_data(ptr, self.size)
         raise self._convert_error("unicode string of length 1", w_ob)
 
     def convert_from_object(self, cdata, w_ob):
-        value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        ordinal = self._convert_to_charN_t(w_ob)
+        misc.write_raw_unsigned_data(cdata, ordinal, self.size)
 
     def unpack_ptr(self, w_ctypeptr, ptr, length):
-        u = rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+        if self.size == 2:
+            u = wchar_helper.unicode_from_char16(ptr, length)
+        else:
+            try:
+                u = wchar_helper.unicode_from_char32(ptr, length)
+            except wchar_helper.OutOfRange as e:
+                raise oefmt(self.space.w_ValueError,
+                            "char32_t out of range for "
+                            "conversion to unicode: %s", hex(e.ordinal))
         return self.space.newunicode(u)
 
 
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -6,9 +6,9 @@
 
 from rpython.rlib import rposix
 from rpython.rlib.rarithmetic import ovfcheck
-from rpython.rtyper.annlowlevel import llstr, llunicode
+from rpython.rtyper.annlowlevel import llstr
 from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
+from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw
 
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
@@ -90,18 +90,28 @@
             if n != self.length:
                 cdata[n] = '\x00'
         elif isinstance(self.ctitem, ctypeprim.W_CTypePrimitiveUniChar):
+            from pypy.module._cffi_backend import wchar_helper
             if not space.isinstance_w(w_ob, space.w_unicode):
                 raise self._convert_error("unicode or list or tuple", w_ob)
             s = space.unicode_w(w_ob)
-            n = len(s)
+            if self.ctitem.size == 2:
+                n = wchar_helper.unicode_size_as_char16(s)
+            else:
+                n = wchar_helper.unicode_size_as_char32(s)
             if self.length >= 0 and n > self.length:
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
-            if n != self.length:
-                unichardata[n] = u'\x00'
+            add_final_zero = (n != self.length)
+            if self.ctitem.size == 2:
+                try:
+                    wchar_helper.unicode_to_char16(s, cdata, n, add_final_zero)
+                except wchar_helper.OutOfRange as e:
+                    raise oefmt(self.space.w_ValueError,
+                                "unicode character ouf of range for "
+                                "conversion to char16_t: %s", hex(e.ordinal))
+            else:
+                wchar_helper.unicode_to_char32(s, cdata, n, add_final_zero)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
@@ -136,12 +146,12 @@
                 #
                 # pointer to a wchar_t: builds and returns a unicode
                 if self.is_unichar_ptr_or_array():
-                    cdata = rffi.cast(rffi.CWCHARP, ptr)
-                    if length < 0:
-                        u = rffi.wcharp2unicode(cdata)
+                    from pypy.module._cffi_backend import wchar_helper
+                    if self.ctitem.size == 2:
+                        length = wchar_helper.measure_length_16(ptr, length)
                     else:
-                        u = rffi.wcharp2unicoden(cdata, length)
-                    return space.newunicode(u)
+                        length = wchar_helper.measure_length_32(ptr, length)
+                    return self.ctitem.unpack_ptr(self, ptr, length)
         #
         return W_CType.string(self, cdataobj, maxlen)
 
@@ -304,10 +314,18 @@
         if (space.isinstance_w(w_init, space.w_list) or
             space.isinstance_w(w_init, space.w_tuple)):
             length = space.int_w(space.len(w_init))
-        elif (space.isinstance_w(w_init, space.w_unicode) or
-              space.isinstance_w(w_init, space.w_bytes)):
+        elif space.isinstance_w(w_init, space.w_bytes):
             # from a string, we add the null terminator
-            length = space.int_w(space.len(w_init)) + 1
+            s = space.bytes_w(w_init)
+            length = len(s) + 1
+        elif space.isinstance_w(w_init, space.w_unicode):
+            from pypy.module._cffi_backend import wchar_helper
+            u = space.unicode_w(w_init)
+            if self.ctitem.size == 2:
+                length = wchar_helper.unicode_size_as_char16(u)
+            else:
+                length = wchar_helper.unicode_size_as_char32(u)
+            length += 1
         elif self.is_file:
             result = self.prepare_file(w_init)
             if result:
diff --git a/pypy/module/_cffi_backend/ctypestruct.py b/pypy/module/_cffi_backend/ctypestruct.py
--- a/pypy/module/_cffi_backend/ctypestruct.py
+++ b/pypy/module/_cffi_backend/ctypestruct.py
@@ -244,7 +244,7 @@
         ct = self.ctype
         if isinstance(ct, ctypearray.W_CTypeArray) and ct.length < 0:
             space = ct.space
-            w_ob, varsizelength = misc.get_new_array_length(space, w_ob)
+            w_ob, varsizelength = ct.get_new_array_length(w_ob)
             if optvarsize != -1:
                 # in this mode, the only purpose of this function is to compute
                 # the real size of the structure from a var-sized C99 array
diff --git a/pypy/module/_cffi_backend/misc.py b/pypy/module/_cffi_backend/misc.py
--- a/pypy/module/_cffi_backend/misc.py
+++ b/pypy/module/_cffi_backend/misc.py
@@ -294,22 +294,6 @@
 
 # ____________________________________________________________
 
-def get_new_array_length(space, w_value):
-    if (space.isinstance_w(w_value, space.w_list) or
-        space.isinstance_w(w_value, space.w_tuple)):
-        return (w_value, space.int_w(space.len(w_value)))
-    elif (space.isinstance_w(w_value, space.w_unicode) or
-          space.isinstance_w(w_value, space.w_bytes)):
-        # from a string, we add the null terminator
-        return (w_value, space.int_w(space.len(w_value)) + 1)
-    else:
-        explicitlength = space.getindex_w(w_value, space.w_OverflowError)
-        if explicitlength < 0:
-            raise oefmt(space.w_ValueError, "negative array length")
-        return (space.w_None, explicitlength)
-
-# ____________________________________________________________
-
 @specialize.arg(0)
 def _raw_memcopy_tp(TPP, source, dest):
     # in its own function: LONGLONG may make the whole function jit-opaque
diff --git a/pypy/module/_cffi_backend/newtype.py b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -111,6 +111,9 @@
 eptype("size_t",    rffi.SIZE_T,    ctypeprim.W_CTypePrimitiveUnsigned)
 eptype("ssize_t",   rffi.SSIZE_T,   ctypeprim.W_CTypePrimitiveSigned)
 
+eptypesize("char16_t", 2, ctypeprim.W_CTypePrimitiveUniChar)
+eptypesize("char32_t", 4, ctypeprim.W_CTypePrimitiveUniChar)
+
 _WCTSigned = ctypeprim.W_CTypePrimitiveSigned
 _WCTUnsign = ctypeprim.W_CTypePrimitiveUnsigned
 
diff --git a/pypy/module/_cffi_backend/realize_c_type.py b/pypy/module/_cffi_backend/realize_c_type.py
--- a/pypy/module/_cffi_backend/realize_c_type.py
+++ b/pypy/module/_cffi_backend/realize_c_type.py
@@ -73,6 +73,8 @@
         "uintmax_t",
         "float _Complex",
         "double _Complex",
+        "char16_t",
+        "char32_t",
         ]
     assert len(NAMES) == cffi_opcode._NUM_PRIM
 
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.c b/pypy/module/_cffi_backend/src/parse_c_type.c
--- a/pypy/module/_cffi_backend/src/parse_c_type.c
+++ b/pypy/module/_cffi_backend/src/parse_c_type.c
@@ -505,6 +505,7 @@
 
     case '1':
         if (size == 8 && !memcmp(p, "uint16", 6)) return _CFFI_PRIM_UINT16;
+        if (size == 8 && !memcmp(p, "char16", 6)) return _CFFI_PRIM_CHAR16;
         break;
 
     case '2':
@@ -513,6 +514,7 @@
 
     case '3':
         if (size == 8 && !memcmp(p, "uint32", 6)) return _CFFI_PRIM_UINT32;
+        if (size == 8 && !memcmp(p, "char32", 6)) return _CFFI_PRIM_CHAR32;
         break;
 
     case '4':
diff --git a/pypy/module/_cffi_backend/src/parse_c_type.h b/pypy/module/_cffi_backend/src/parse_c_type.h
--- a/pypy/module/_cffi_backend/src/parse_c_type.h
+++ b/pypy/module/_cffi_backend/src/parse_c_type.h
@@ -80,8 +80,10 @@
 #define _CFFI_PRIM_UINTMAX      47
 #define _CFFI_PRIM_FLOATCOMPLEX 48
 #define _CFFI_PRIM_DOUBLECOMPLEX 49
+#define _CFFI_PRIM_CHAR16       50
+#define _CFFI_PRIM_CHAR32       51
 
-#define _CFFI__NUM_PRIM         50
+#define _CFFI__NUM_PRIM         52
 #define _CFFI__UNKNOWN_PRIM           (-1)
 #define _CFFI__UNKNOWN_FLOAT_PRIM     (-2)
 #define _CFFI__UNKNOWN_LONG_DOUBLE    (-3)
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -1925,7 +1925,11 @@
         assert string(a, 8).startswith(b'ABC')  # may contain additional garbage
 
 def test_string_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        _test_string_wchar_variant(typename)
+
+def _test_string_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     assert string(cast(BWChar, 42)) == u+'*'
     assert string(cast(BWChar, 0x4253)) == u+'\u4253'
     assert string(cast(BWChar, 0)) == u+'\x00'
@@ -2087,22 +2091,44 @@
     py.test.raises(TypeError, newp, BStructPtr, [cast(BFunc2, 0)])
 
 def test_wchar():
-    BWChar = new_primitive_type("wchar_t")
+    _test_wchar_variant("wchar_t")
+    if sys.platform.startswith("linux"):
+        BWChar = new_primitive_type("wchar_t")
+        assert sizeof(BWChar) == 4
+        assert int(cast(BWChar, -1)) == -1        # signed, on linux
+
+def test_char16():
+    BChar16 = new_primitive_type("char16_t")
+    assert sizeof(BChar16) == 2
+    _test_wchar_variant("char16_t")
+    assert int(cast(BChar16, -1)) == 0xffff       # always unsigned
+
+def test_char32():
+    BChar32 = new_primitive_type("char32_t")
+    assert sizeof(BChar32) == 4
+    _test_wchar_variant("char32_t")
+    assert int(cast(BChar32, -1)) == 0xffffffff   # always unsigned
+
+def _test_wchar_variant(typename):
+    BWChar = new_primitive_type(typename)
     BInt = new_primitive_type("int")
     pyuni4 = {1: True, 2: False}[len(u+'\U00012345')]
     wchar4 = {2: False, 4: True}[sizeof(BWChar)]
-    assert str(cast(BWChar, 0x45)) == "<cdata 'wchar_t' %s'E'>" % (
-        mandatory_u_prefix,)
-    assert str(cast(BWChar, 0x1234)) == "<cdata 'wchar_t' %s'\u1234'>" % (
-        mandatory_u_prefix,)
-    if wchar4:
-        if not _hacked_pypy_uni4():
+    assert str(cast(BWChar, 0x45)) == "<cdata '%s' %s'E'>" % (
+        typename, mandatory_u_prefix)
+    assert str(cast(BWChar, 0x1234)) == "<cdata '%s' %s'\u1234'>" % (
+        typename, mandatory_u_prefix)
+    if not _hacked_pypy_uni4():
+        if wchar4:
             x = cast(BWChar, 0x12345)
-            assert str(x) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-                mandatory_u_prefix,)
+            assert str(x) == "<cdata '%s' %s'\U00012345'>" % (
+                typename, mandatory_u_prefix)
             assert int(x) == 0x12345
-    else:
-        assert not pyuni4
+        else:
+            x = cast(BWChar, 0x18345)
+            assert str(x) == "<cdata '%s' %s'\u8345'>" % (
+                typename, mandatory_u_prefix)
+            assert int(x) == 0x8345
     #
     BWCharP = new_pointer_type(BWChar)
     BStruct = new_struct_type("struct foo_s")
@@ -2117,9 +2143,9 @@
     s.a1 = u+'\u1234'
     assert s.a1 == u+'\u1234'
     if pyuni4:
-        assert wchar4
-        s.a1 = u+'\U00012345'
-        assert s.a1 == u+'\U00012345'
+        if wchar4:
+            s.a1 = u+'\U00012345'
+            assert s.a1 == u+'\U00012345'
     elif wchar4:
         if not _hacked_pypy_uni4():
             s.a1 = cast(BWChar, 0x12345)
@@ -2154,17 +2180,17 @@
         py.test.raises(IndexError, 'a[4]')
     #
     w = cast(BWChar, 'a')
-    assert repr(w) == "<cdata 'wchar_t' %s'a'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'a'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'a'
     assert int(w) == ord('a')
     w = cast(BWChar, 0x1234)
-    assert repr(w) == "<cdata 'wchar_t' %s'\u1234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u1234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u1234'
     assert int(w) == 0x1234
     w = cast(BWChar, u+'\u8234')
-    assert repr(w) == "<cdata 'wchar_t' %s'\u8234'>" % mandatory_u_prefix
+    assert repr(w) == "<cdata '%s' %s'\u8234'>" % (typename, mandatory_u_prefix)
     assert str(w) == repr(w)
     assert string(w) == u+'\u8234'
     assert int(w) == 0x8234
@@ -2172,8 +2198,8 @@
     assert repr(w) == "<cdata 'int' 4660>"
     if wchar4 and not _hacked_pypy_uni4():
         w = cast(BWChar, u+'\U00012345')
-        assert repr(w) == "<cdata 'wchar_t' %s'\U00012345'>" % (
-            mandatory_u_prefix,)
+        assert repr(w) == "<cdata '%s' %s'\U00012345'>" % (
+            typename, mandatory_u_prefix)
         assert str(w) == repr(w)
         assert string(w) == u+'\U00012345'
         assert int(w) == 0x12345
@@ -2200,7 +2226,7 @@
     py.test.raises(RuntimeError, string, q)
     #
     def cb(p):
-        assert repr(p).startswith("<cdata 'wchar_t *' 0x")
+        assert repr(p).startswith("<cdata '%s *' 0x" % typename)
         return len(string(p))
     BFunc = new_function_type((BWCharP,), BInt, False)
     f = callback(BFunc, cb, -42)
@@ -2213,6 +2239,27 @@
         x = cast(BWChar, -1)
         py.test.raises(ValueError, string, x)
 
+def test_wchar_variants_mix():
+    BWChar  = new_primitive_type("wchar_t")
+    BChar16 = new_primitive_type("char16_t")
+    BChar32 = new_primitive_type("char32_t")
+    assert int(cast(BChar32, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BWChar, cast(BChar16, -2))) == 0xfffe
+    assert int(cast(BChar16, cast(BChar32, 0x0001f345))) == 0xf345
+    assert int(cast(BChar16, cast(BWChar, 0x0001f345))) == 0xf345
+    #
+    BChar16A = new_array_type(new_pointer_type(BChar16), None)
+    BChar32A = new_array_type(new_pointer_type(BChar32), None)
+    x = cast(BChar32, 'A')
+    py.test.raises(TypeError, newp, BChar16A, [x])
+    x = cast(BChar16, 'A')
+    py.test.raises(TypeError, newp, BChar32A, [x])
+    #
+    a = newp(BChar16A, u+'\U00012345')
+    assert len(a) == 3
+    a = newp(BChar32A, u+'\U00012345')
+    assert len(a) == 2   # even if the Python unicode string above is 2 chars
+
 def test_keepalive_struct():
     # exception to the no-keepalive rule: p=newp(BStructPtr) returns a
     # pointer owning the memory, and p[0] returns a pointer to the
@@ -3439,14 +3486,15 @@
     py.test.raises(TypeError, "p[1:5] = u+'XYZT'")
     py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
     #
-    BUniChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BUniChar), None)
-    p = newp(BArray, u+"foobar")
-    p[2:5] = [u+"*", u+"Z", u+"T"]
-    p[1:3] = u+"XY"
-    assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
-    py.test.raises(TypeError, "p[1:5] = b'XYZT'")
-    py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BUniChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BUniChar), None)
+        p = newp(BArray, u+"foobar")
+        p[2:5] = [u+"*", u+"Z", u+"T"]
+        p[1:3] = u+"XY"
+        assert list(p) == [u+"f", u+"X", u+"Y", u+"Z", u+"T", u+"r", u+"\x00"]
+        py.test.raises(TypeError, "p[1:5] = b'XYZT'")
+        py.test.raises(TypeError, "p[1:5] = [1, 2, 3, 4]")
 
 def test_void_p_arithmetic():
     BVoid = new_void_type()
@@ -3759,10 +3807,12 @@
     p0 = p
     assert unpack(p, 10) == b"abc\x00def\x00\x00\x00"
     assert unpack(p+1, 5) == b"bc\x00de"
-    BWChar = new_primitive_type("wchar_t")
-    BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
-    p = newp(BArray, u"abc\x00def")
-    assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
+
+    for typename in ["wchar_t", "char16_t", "char32_t"]:
+        BWChar = new_primitive_type(typename)
+        BArray = new_array_type(new_pointer_type(BWChar), 10)   # wchar_t[10]
+        p = newp(BArray, u"abc\x00def")
+        assert unpack(p, 10) == u"abc\x00def\x00\x00\x00"
 
     for typename, samples in [
             ("uint8_t",  [0, 2**8-1]),
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -555,3 +555,11 @@
         import _cffi_backend as _cffi1_backend
         ffi = _cffi1_backend.FFI()
         raises(ffi.error, ffi.cast, "int[-5]", 0)
+
+    def test_char32_t(self):
+        import _cffi_backend as _cffi1_backend
+        ffi = _cffi1_backend.FFI()
+        z = ffi.new("char32_t[]", u'\U00012345')
+        assert len(z) == 2
+        assert ffi.cast("int *", z)[0] == 0x12345
+        assert list(z) == [u'\U00012345', u'\x00']   # maybe a 2-unichars str
diff --git a/pypy/module/_cffi_backend/wchar_helper.py b/pypy/module/_cffi_backend/wchar_helper.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/wchar_helper.py
@@ -0,0 +1,192 @@
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import r_uint, r_ulonglong, intmask
+from rpython.rtyper.annlowlevel import llunicode
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rtyper.lltypesystem.rstr import copy_unicode_to_raw
+
+SIZE_UNICODE = rffi.sizeof(lltype.UniChar)
+
+
+if SIZE_UNICODE == 4:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        return unichr(intmask(ordinal))
+else:
+    def ordinal_to_unicode(ordinal):    # 'ordinal' is a r_uint
+        if ordinal <= 0xffff:
+            return unichr(intmask(ordinal))
+        elif ordinal <= 0x10ffff:
+            ordinal = intmask(ordinal - 0x10000)
+            return (unichr(0xD800 | (ordinal >> 10)) +
+                    unichr(0xDC00 | (ordinal & 0x3FF)))
+        else:
+            raise OutOfRange(ordinal)
+
+def is_surrogate(u, index):
+    return (unichr(0xD800) <= u[index + 0] <= unichr(0xDBFF) and
+            unichr(0xDC00) <= u[index + 1] <= unichr(0xDFFF))
+
+def as_surrogate(u, index):
+    ordinal = (ord(u[index + 0]) - 0xD800) << 10
+    ordinal |= (ord(u[index + 1]) - 0xDC00)
+    return r_uint(ordinal + 0x10000)
+
+def unicode_to_ordinal(u):
+    if len(u) == 1:
+        u = ord(u[0])
+        return r_uint(u)
+    elif SIZE_UNICODE == 2:
+        if len(u) == 2 and is_surrogate(u, 0):
+            return r_uint(as_surrogate(u, 0))
+    raise ValueError
+
+
+class OutOfRange(Exception):
+    ordinal = 0
+
+    def __init__(self, ordinal):
+        ordinal = intmask(rffi.cast(rffi.INT, ordinal))
+        self.ordinal = ordinal
+
+def _unicode_from_wchar(ptr, length):
+    return rffi.wcharpsize2unicode(rffi.cast(rffi.CWCHARP, ptr), length)
+
+
+if SIZE_UNICODE == 2:
+    def unicode_from_char32(ptr, length):
+        # 'ptr' is a pointer to 'length' 32-bit integers
+        ptr = rffi.cast(rffi.UINTP, ptr)
+        alloc = length
+        for i in range(length):
+            if rffi.cast(lltype.Unsigned, ptr[i]) > 0xFFFF:
+                alloc += 1
+
+        u = [u'\x00'] * alloc
+        j = 0
+        for i in range(length):
+            ordinal = rffi.cast(lltype.Unsigned, ptr[i])
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal = intmask(ordinal - 0x10000)
+                u[j] = unichr(0xD800 | (ordinal >> 10))
+                j += 1
+                u[j] = unichr(0xDC00 | (ordinal & 0x3FF))
+                j += 1
+            else:
+                u[j] = unichr(intmask(ordinal))
+                j += 1
+        assert j == len(u)
+        return u''.join(u)
+
+    unicode_from_char16 = _unicode_from_wchar
+
+else:
+    unicode_from_char32 = _unicode_from_wchar
+
+    def unicode_from_char16(ptr, length):
+        # 'ptr' is a pointer to 'length' 16-bit integers
+        ptr = rffi.cast(rffi.USHORTP, ptr)
+        u = [u'\x00'] * length
+        i = 0
+        j = 0
+        while j < length:
+            ch = intmask(ptr[j])
+            j += 1
+            if 0xD800 <= ch <= 0xDBFF and j < length:
+                ch2 = intmask(ptr[j])
+                if 0xDC00 <= ch2 <= 0xDFFF:
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000
+                    j += 1
+            u[i] = unichr(ch)
+            i += 1
+        del u[i:]
+        return u''.join(u)
+
+
+ at specialize.ll()
+def _measure_length(ptr, maxlen):
+    result = 0
+    if maxlen < 0:
+        while intmask(ptr[result]) != 0:
+            result += 1
+    else:
+        while result < maxlen and intmask(ptr[result]) != 0:
+            result += 1
+    return result
+
+def measure_length_16(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.USHORTP, ptr), maxlen)
+
+def measure_length_32(ptr, maxlen=-1):
+    return _measure_length(rffi.cast(rffi.UINTP, ptr), maxlen)
+
+
+def unicode_size_as_char16(u):
+    result = len(u)
+    if SIZE_UNICODE == 4:
+        for i in range(result):
+            if ord(u[i]) > 0xFFFF:
+                result += 1
+    return result
+
+def unicode_size_as_char32(u):
+    result = len(u)
+    if SIZE_UNICODE == 2 and result > 1:
+        for i in range(result - 1):
+            if is_surrogate(u, i):
+                result -= 1
+    return result
+
+
+def _unicode_to_wchar(u, target_ptr, target_length, add_final_zero):
+    # 'target_ptr' is a raw pointer to 'target_length' wchars;
+    # we assume here that target_length == len(u).
+    unichardata = rffi.cast(rffi.CWCHARP, target_ptr)
+    copy_unicode_to_raw(llunicode(u), unichardata, 0, target_length)
+    if add_final_zero:
+        unichardata[target_length] = u'\x00'
+
+
+if SIZE_UNICODE == 2:
+    def unicode_to_char32(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 32-bit integers;
+        # we assume here that target_length == unicode_size_as_char32(u).
+        ptr = rffi.cast(rffi.UINTP, target_ptr)
+        src_index = 0
+        last_surrogate_pos = len(u) - 2
+        for i in range(target_length):
+            if src_index <= last_surrogate_pos and is_surrogate(u, src_index):
+                ordinal = as_surrogate(u, src_index)
+                src_index += 2
+            else:
+                ordinal = r_uint(ord(u[src_index]))
+                src_index += 1
+            ptr[i] = rffi.cast(rffi.UINT, ordinal)
+        if add_final_zero:
+            ptr[target_length] = rffi.cast(rffi.UINT, 0)
+
+    unicode_to_char16 = _unicode_to_wchar
+
+else:
+    unicode_to_char32 = _unicode_to_wchar
+
+    def unicode_to_char16(u, target_ptr, target_length, add_final_zero):
+        # 'target_ptr' is a raw pointer to 'target_length' 16-bit integers;
+        # we assume here that target_length == unicode_size_as_char16(u).
+        ptr = rffi.cast(rffi.USHORTP, target_ptr)
+        for uc in u:
+            ordinal = ord(uc)
+            if ordinal > 0xFFFF:
+                if ordinal > 0x10FFFF:
+                    raise OutOfRange(ordinal)
+                ordinal -= 0x10000
+                ptr[0] = rffi.cast(rffi.USHORT, 0xD800 | (ordinal >> 10))
+                ptr[1] = rffi.cast(rffi.USHORT, 0xDC00 | (ordinal & 0x3FF))
+                ptr = rffi.ptradd(ptr, 2)
+            else:
+                ptr[0] = rffi.cast(rffi.USHORT, ordinal)
+                ptr = rffi.ptradd(ptr, 1)
+        assert ptr == (
+            rffi.ptradd(rffi.cast(rffi.USHORTP, target_ptr), target_length))
+        if add_final_zero:
+            ptr[0] = rffi.cast(rffi.USHORT, 0)
diff --git a/pypy/module/_vmprof/__init__.py b/pypy/module/_vmprof/__init__.py
--- a/pypy/module/_vmprof/__init__.py
+++ b/pypy/module/_vmprof/__init__.py
@@ -14,6 +14,9 @@
         'write_all_code_objects': 'interp_vmprof.write_all_code_objects',
         'is_enabled': 'interp_vmprof.is_enabled',
         'get_profile_path': 'interp_vmprof.get_profile_path',
+        'stop_sampling': 'interp_vmprof.stop_sampling',
+        'start_sampling': 'interp_vmprof.start_sampling',
+
         'VMProfError': 'space.fromcache(interp_vmprof.Cache).w_VMProfError',
     }
 
diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py
--- a/pypy/module/_vmprof/interp_vmprof.py
+++ b/pypy/module/_vmprof/interp_vmprof.py
@@ -51,8 +51,8 @@
     return OperationError(w_VMProfError, space.newtext(e.msg))
 
 
- at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int)
-def enable(space, fileno, period, memory, lines, native):
+ at unwrap_spec(fileno=int, period=float, memory=int, lines=int, native=int, real_time=int)
+def enable(space, fileno, period, memory, lines, native, real_time):
     """Enable vmprof.  Writes go to the given 'fileno', a file descriptor
     opened for writing.  *The file descriptor must remain open at least
     until disable() is called.*
@@ -66,7 +66,7 @@
     #                             "with vmprof will crash"),
     #               space.w_RuntimeWarning)
     try:
-        rvmprof.enable(fileno, period, memory, native)
+        rvmprof.enable(fileno, period, memory, native, real_time)
     except rvmprof.VMProfError as e:
         raise VMProfError(space, e)
 
@@ -96,3 +96,10 @@

From pypy.commits at gmail.com  Sun Jun 11 17:22:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 14:22:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: fixes
Message-ID: <593db4a3.4495df0a.57b67.f9b9@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91578:d936612da330
Date: 2017-06-11 23:16 +0200
http://bitbucket.org/pypy/pypy/changeset/d936612da330/

Log:	fixes

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -711,9 +711,9 @@
 
         with lltype.scoped_alloc(
                 win32traits.BY_HANDLE_FILE_INFORMATION) as data:
-            res = rwin32.GetFileInformationByHandle(hFile, data)
+            res = win32traits.GetFileInformationByHandle(hFile, data)
             errcode = rwin32.GetLastError_saved()
-            win32.CloseHandle(hFile)
+            rwin32.CloseHandle(hFile)
             if res == 0:
                 raise WindowsError(errcode, "GetFileInformationByHandle failed")
             return win32_by_handle_info_to_stat(win32traits, data)
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -109,7 +109,7 @@
                        INVALID_FILE_ATTRIBUTES
                        _S_IFDIR _S_IFREG _S_IFCHR _S_IFIFO
                        FILE_TYPE_UNKNOWN FILE_TYPE_CHAR FILE_TYPE_PIPE
-                       FILE_READ_ATTRIBUTES
+                       FILE_READ_ATTRIBUTES FILE_ATTRIBUTE_NORMAL
                        FILE_WRITE_ATTRIBUTES OPEN_EXISTING FILE_FLAG_BACKUP_SEMANTICS
                        VOLUME_NAME_DOS VOLUME_NAME_NT
                        ERROR_FILE_NOT_FOUND ERROR_NO_MORE_FILES
diff --git a/rpython/rlib/test/test_rposix_stat.py b/rpython/rlib/test/test_rposix_stat.py
--- a/rpython/rlib/test/test_rposix_stat.py
+++ b/rpython/rlib/test/test_rposix_stat.py
@@ -70,11 +70,11 @@
 
     @py.test.mark.skipif(sys.platform != 'win32', reason='win32 test')
     def test_stat3_ino_dev(self):
-        st = rposix.stat2('C:\\')
+        st = rposix_stat.stat('C:\\')
         assert st.st_dev == st.st_ino == 0
-        st = rposix.stat3('C:\\')
+        st = rposix_stat.stat3('C:\\')
         assert st.st_dev != 0 and st.st_ino != 0
-        st2 = rposix.lstat3('C:\\')
+        st2 = rposix_stat.lstat3('C:\\')
         assert (st2.st_dev, st2.st_ino) == (st.st_dev, st.st_ino)
 
 

From pypy.commits at gmail.com  Sun Jun 11 17:22:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 14:22:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Update posix to call the new stat3()
 instead of stat() in py3.5
Message-ID: <593db4a7.5888df0a.56975.90e2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91580:84f77db110c0
Date: 2017-06-11 23:22 +0200
http://bitbucket.org/pypy/pypy/changeset/84f77db110c0/

Log:	Update posix to call the new stat3() instead of stat() in py3.5

diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -531,9 +531,9 @@
                     "%s: cannot use fd and follow_symlinks together", funcname)
             st = rposix_stat.fstat(path.as_fd)
         elif follow_symlinks and dir_fd == DEFAULT_DIR_FD:
-            st = call_rposix(rposix_stat.stat, path)
+            st = call_rposix(rposix_stat.stat3, path)
         elif not follow_symlinks and dir_fd == DEFAULT_DIR_FD:
-            st = call_rposix(rposix_stat.lstat, path)
+            st = call_rposix(rposix_stat.lstat3, path)
         elif rposix.HAVE_FSTATAT:
             st = call_rposix(rposix_stat.fstatat, path, dir_fd, follow_symlinks)
         else:

From pypy.commits at gmail.com  Sun Jun 11 18:06:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 15:06:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: translation fixes
Message-ID: <593dbefb.430d1c0a.37f3d.fbc5@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91581:617de56df11b
Date: 2017-06-11 23:43 +0200
http://bitbucket.org/pypy/pypy/changeset/617de56df11b/

Log:	translation fixes

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -710,7 +710,7 @@
                 return win32_attribute_data_to_stat(win32traits, data)
 
         with lltype.scoped_alloc(
-                win32traits.BY_HANDLE_FILE_INFORMATION) as data:
+                win32traits.BY_HANDLE_FILE_INFORMATION, zero=True) as data:
             res = win32traits.GetFileInformationByHandle(hFile, data)
             errcode = rwin32.GetLastError_saved()
             rwin32.CloseHandle(hFile)
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -8,29 +8,14 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import intmask
 
- at specialize.memo()
-def make_win32_traits(traits):
+
+def GetCConfigGlobal():
     from rpython.rlib import rwin32
 
-    if traits.str is unicode:
-        suffix = 'W'
-    else:
-        suffix = 'A'
-
-    class CConfig:
+    class CConfigGlobal:
         _compilation_info_ = ExternalCompilationInfo(
             includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
         )
-        WIN32_FIND_DATA = platform.Struct(
-            'struct _WIN32_FIND_DATA' + suffix,
-            # Only interesting fields
-            [('dwFileAttributes', rwin32.DWORD),
-             ('nFileSizeHigh', rwin32.DWORD),
-             ('nFileSizeLow', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME),
-             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
         ERROR_FILE_NOT_FOUND = platform.ConstantInteger(
             'ERROR_FILE_NOT_FOUND')
         ERROR_NO_MORE_FILES = platform.ConstantInteger(
@@ -93,7 +78,40 @@
              ('nFileIndexHigh', rwin32.DWORD),
              ('nFileIndexLow', rwin32.DWORD)])
 
-    config = platform.configure(CConfig)
+    return CConfigGlobal
+
+config_global = None
+
+
+ at specialize.memo()
+def make_win32_traits(traits):
+    from rpython.rlib import rwin32
+    global config_global
+
+    if traits.str is unicode:
+        suffix = 'W'
+    else:
+        suffix = 'A'
+
+    class CConfig:
+        _compilation_info_ = ExternalCompilationInfo(
+            includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
+        )
+        WIN32_FIND_DATA = platform.Struct(
+            'struct _WIN32_FIND_DATA' + suffix,
+            # Only interesting fields
+            [('dwFileAttributes', rwin32.DWORD),
+             ('nFileSizeHigh', rwin32.DWORD),
+             ('nFileSizeLow', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME),
+             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
+
+    if config_global is None:
+        config_global = platform.configure(GetCConfigGlobal())
+    config = config_global.copy()
+    config.update(platform.configure(CConfig))
 
     def external(*args, **kwargs):
         kwargs['compilation_info'] = CConfig._compilation_info_

From pypy.commits at gmail.com  Sun Jun 11 18:06:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 15:06:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <593dbf00.10d91c0a.6925f.63a3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91584:c58d210eb947
Date: 2017-06-12 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/c58d210eb947/

Log:	hg merge default

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -748,6 +748,7 @@
 
         return make_stat_result(result)
 
+    @specialize.arg(0)
     def win32_by_handle_info_to_stat(win32traits, info):
         # similar to the one above
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)

From pypy.commits at gmail.com  Sun Jun 11 18:06:53 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 15:06:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <593dbefd.c48d1c0a.cb041.91f0@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91582:fd48d2029916
Date: 2017-06-11 23:43 +0200
http://bitbucket.org/pypy/pypy/changeset/fd48d2029916/

Log:	hg merge default

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -710,7 +710,7 @@
                 return win32_attribute_data_to_stat(win32traits, data)
 
         with lltype.scoped_alloc(
-                win32traits.BY_HANDLE_FILE_INFORMATION) as data:
+                win32traits.BY_HANDLE_FILE_INFORMATION, zero=True) as data:
             res = win32traits.GetFileInformationByHandle(hFile, data)
             errcode = rwin32.GetLastError_saved()
             rwin32.CloseHandle(hFile)
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -8,29 +8,14 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.rarithmetic import intmask
 
- at specialize.memo()
-def make_win32_traits(traits):
+
+def GetCConfigGlobal():
     from rpython.rlib import rwin32
 
-    if traits.str is unicode:
-        suffix = 'W'
-    else:
-        suffix = 'A'
-
-    class CConfig:
+    class CConfigGlobal:
         _compilation_info_ = ExternalCompilationInfo(
             includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
         )
-        WIN32_FIND_DATA = platform.Struct(
-            'struct _WIN32_FIND_DATA' + suffix,
-            # Only interesting fields
-            [('dwFileAttributes', rwin32.DWORD),
-             ('nFileSizeHigh', rwin32.DWORD),
-             ('nFileSizeLow', rwin32.DWORD),
-             ('ftCreationTime', rwin32.FILETIME),
-             ('ftLastAccessTime', rwin32.FILETIME),
-             ('ftLastWriteTime', rwin32.FILETIME),
-             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
         ERROR_FILE_NOT_FOUND = platform.ConstantInteger(
             'ERROR_FILE_NOT_FOUND')
         ERROR_NO_MORE_FILES = platform.ConstantInteger(
@@ -93,7 +78,40 @@
              ('nFileIndexHigh', rwin32.DWORD),
              ('nFileIndexLow', rwin32.DWORD)])
 
-    config = platform.configure(CConfig)
+    return CConfigGlobal
+
+config_global = None
+
+
+ at specialize.memo()
+def make_win32_traits(traits):
+    from rpython.rlib import rwin32
+    global config_global
+
+    if traits.str is unicode:
+        suffix = 'W'
+    else:
+        suffix = 'A'
+
+    class CConfig:
+        _compilation_info_ = ExternalCompilationInfo(
+            includes = ['windows.h', 'winbase.h', 'sys/stat.h'],
+        )
+        WIN32_FIND_DATA = platform.Struct(
+            'struct _WIN32_FIND_DATA' + suffix,
+            # Only interesting fields
+            [('dwFileAttributes', rwin32.DWORD),
+             ('nFileSizeHigh', rwin32.DWORD),
+             ('nFileSizeLow', rwin32.DWORD),
+             ('ftCreationTime', rwin32.FILETIME),
+             ('ftLastAccessTime', rwin32.FILETIME),
+             ('ftLastWriteTime', rwin32.FILETIME),
+             ('cFileName', lltype.FixedSizeArray(traits.CHAR, 250))])
+
+    if config_global is None:
+        config_global = platform.configure(GetCConfigGlobal())
+    config = config_global.copy()
+    config.update(platform.configure(CConfig))
 
     def external(*args, **kwargs):
         kwargs['compilation_info'] = CConfig._compilation_info_

From pypy.commits at gmail.com  Sun Jun 11 18:06:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 15:06:54 -0700 (PDT)
Subject: [pypy-commit] pypy default: translation fix
Message-ID: <593dbefe.10d91c0a.6925f.63a2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91583:877f60c21755
Date: 2017-06-12 00:06 +0200
http://bitbucket.org/pypy/pypy/changeset/877f60c21755/

Log:	translation fix

diff --git a/rpython/rlib/rposix_stat.py b/rpython/rlib/rposix_stat.py
--- a/rpython/rlib/rposix_stat.py
+++ b/rpython/rlib/rposix_stat.py
@@ -748,6 +748,7 @@
 
         return make_stat_result(result)
 
+    @specialize.arg(0)
     def win32_by_handle_info_to_stat(win32traits, info):
         # similar to the one above
         st_mode = win32_attributes_to_mode(win32traits, info.c_dwFileAttributes)

From pypy.commits at gmail.com  Mon Jun 12 01:21:51 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 22:21:51 -0700 (PDT)
Subject: [pypy-commit] pypy default: _curses: Export more A_* constants.
 (from linkmauve)
Message-ID: <593e24ef.81851c0a.7634.bae4@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91585:389ed2794e24
Date: 2017-06-12 07:21 +0200
http://bitbucket.org/pypy/pypy/changeset/389ed2794e24/

Log:	_curses: Export more A_* constants. (from linkmauve)

	These six were already checked for but never imported from C.

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -87,6 +87,13 @@
 static const chtype A_CHARTEXT;
 static const chtype A_COLOR;
 
+static const chtype A_HORIZONTAL;
+static const chtype A_LEFT;
+static const chtype A_LOW;
+static const chtype A_RIGHT;
+static const chtype A_TOP;
+static const chtype A_VERTICAL;
+
 static const int BUTTON1_RELEASED;
 static const int BUTTON1_PRESSED;
 static const int BUTTON1_CLICKED;

From pypy.commits at gmail.com  Mon Jun 12 01:33:50 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 22:33:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: _curses: Implement resizeterm() and
 resize_term().
Message-ID: <593e27be.5888df0a.56975.1d37@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91587:4264cf9b5caf
Date: 2017-06-12 07:32 +0200
http://bitbucket.org/pypy/pypy/changeset/4264cf9b5caf/

Log:	_curses: Implement resizeterm() and resize_term().

	This also implements the update_line_cols() helper function which
	gets the new COLS and LINES values on the Python side.

	(from linkmauve, with fix to also update 'curses.COLS' and
	'curses.LINES')

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -912,101 +912,29 @@
     return None
 
 
-# XXX: Do something about the following?
-# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
-#  * and _curses.COLS */
-# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)
-# static int
-# update_lines_cols(void)
-# {
-#     PyObject *o;
-#     PyObject *m = PyImport_ImportModuleNoBlock("curses");
+# Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
+# and _curses.COLS
+def update_lines_cols():
+    globals()["LINES"] = lib.LINES
+    globals()["COLS"] = lib.COLS
+    try:
+        m = sys.modules["curses"]
+        m.LINES = lib.LINES
+        m.COLS = lib.COLS
+    except (KeyError, AttributeError):
+        pass
 
-#     if (!m)
-#         return 0;
 
-#     o = PyInt_FromLong(LINES);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     o = PyInt_FromLong(COLS);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     Py_DECREF(m);
-#     return 1;
-# }
-# #endif
+def resizeterm(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resizeterm(lines, columns), "resizeterm")
+    update_lines_cols()
 
-# #ifdef HAVE_CURSES_RESIZETERM
-# static PyObject *
-# PyCurses_ResizeTerm(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-#     PyObject *result;
 
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-
-# #endif
-
-# #ifdef HAVE_CURSES_RESIZE_TERM
-# static PyObject *
-# PyCurses_Resize_Term(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-
-#     PyObject *result;
-
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resize_term(lines, columns), "resize_term");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-# #endif /* HAVE_CURSES_RESIZE_TERM */
+def resize_term(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resize_term(lines, columns), "resize_term")
+    update_lines_cols()
 
 
 def setsyx(y, x):
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -209,6 +209,8 @@
 int resetty(void);
 int reset_prog_mode(void);
 int reset_shell_mode(void);
+int resizeterm(int, int);
+int resize_term(int, int);
 int savetty(void);
 int scroll(WINDOW *);
 int scrollok(WINDOW *, bool);

From pypy.commits at gmail.com  Mon Jun 12 01:33:48 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 22:33:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: _curses: Copy the version from cpython.
 (from linkmauve)
Message-ID: <593e27bc.c4581c0a.5941a.7e0a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91586:c2e3d0655f05
Date: 2017-06-12 07:22 +0200
http://bitbucket.org/pypy/pypy/changeset/c2e3d0655f05/

Log:	_curses: Copy the version from cpython. (from linkmauve)

	It is currently b'2.2'.

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -8,6 +8,9 @@
 
 from _curses_cffi import ffi, lib
 
+version = b"2.2"
+__version__ = b"2.2"
+
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
 
@@ -60,10 +63,6 @@
 
 _setup()
 
-# Do we want this?
-# version = "2.2"
-# __version__ = "2.2"
-
 
 # ____________________________________________________________
 

From pypy.commits at gmail.com  Mon Jun 12 01:34:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 22:34:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <593e27ee.88da1c0a.3a0d4.a684@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91588:c3f3e2469a13
Date: 2017-06-12 07:34 +0200
http://bitbucket.org/pypy/pypy/changeset/c3f3e2469a13/

Log:	hg merge default

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -8,6 +8,9 @@
 
 from _curses_cffi import ffi, lib
 
+version = b"2.2"
+__version__ = b"2.2"
+
 def _copy_to_globals(name):
     globals()[name] = getattr(lib, name)
 
@@ -60,10 +63,6 @@
 
 _setup()
 
-# Do we want this?
-# version = "2.2"
-# __version__ = "2.2"
-
 
 # ____________________________________________________________
 
@@ -919,101 +918,29 @@
     return None
 
 
-# XXX: Do something about the following?
-# /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
-#  * and _curses.COLS */
-# #if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)
-# static int
-# update_lines_cols(void)
-# {
-#     PyObject *o;
-#     PyObject *m = PyImport_ImportModuleNoBlock("curses");
+# Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES
+# and _curses.COLS
+def update_lines_cols():
+    globals()["LINES"] = lib.LINES
+    globals()["COLS"] = lib.COLS
+    try:
+        m = sys.modules["curses"]
+        m.LINES = lib.LINES
+        m.COLS = lib.COLS
+    except (KeyError, AttributeError):
+        pass
 
-#     if (!m)
-#         return 0;
 
-#     o = PyInt_FromLong(LINES);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "LINES", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     o = PyInt_FromLong(COLS);
-#     if (!o) {
-#         Py_DECREF(m);
-#         return 0;
-#     }
-#     if (PyObject_SetAttrString(m, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     if (PyDict_SetItemString(ModDict, "COLS", o)) {
-#         Py_DECREF(m);
-#         Py_DECREF(o);
-#         return 0;
-#     }
-#     Py_DECREF(o);
-#     Py_DECREF(m);
-#     return 1;
-# }
-# #endif
+def resizeterm(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resizeterm(lines, columns), "resizeterm")
+    update_lines_cols()
 
-# #ifdef HAVE_CURSES_RESIZETERM
-# static PyObject *
-# PyCurses_ResizeTerm(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-#     PyObject *result;
 
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resizeterm", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resizeterm(lines, columns), "resizeterm");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-
-# #endif
-
-# #ifdef HAVE_CURSES_RESIZE_TERM
-# static PyObject *
-# PyCurses_Resize_Term(PyObject *self, PyObject *args)
-# {
-#     int lines;
-#     int columns;
-
-#     PyObject *result;
-
-#     PyCursesInitialised;
-
-#     if (!PyArg_ParseTuple(args,"ii:resize_term", &lines, &columns))
-#         return NULL;
-
-#     result = PyCursesCheckERR(resize_term(lines, columns), "resize_term");
-#     if (!result)
-#         return NULL;
-#     if (!update_lines_cols())
-#         return NULL;
-#     return result;
-# }
-# #endif /* HAVE_CURSES_RESIZE_TERM */
+def resize_term(lines, columns):
+    _ensure_initialised()
+    _check_ERR(lib.resize_term(lines, columns), "resize_term")
+    update_lines_cols()
 
 
 def setsyx(y, x):
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -105,6 +105,13 @@
 static const chtype A_CHARTEXT;
 static const chtype A_COLOR;
 
+static const chtype A_HORIZONTAL;
+static const chtype A_LEFT;
+static const chtype A_LOW;
+static const chtype A_RIGHT;
+static const chtype A_TOP;
+static const chtype A_VERTICAL;
+
 static const int BUTTON1_RELEASED;
 static const int BUTTON1_PRESSED;
 static const int BUTTON1_CLICKED;
@@ -220,6 +227,8 @@
 int resetty(void);
 int reset_prog_mode(void);
 int reset_shell_mode(void);
+int resizeterm(int, int);
+int resize_term(int, int);
 int savetty(void);
 int scroll(WINDOW *);
 int scrollok(WINDOW *, bool);

From pypy.commits at gmail.com  Mon Jun 12 01:41:26 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 11 Jun 2017 22:41:26 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: _curses: Implement Window.get_wch() and
 unget_wch().
Message-ID: <593e2986.8ab6df0a.1d0b2.2ba2@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91589:223ff388e219
Date: 2017-06-12 07:40 +0200
http://bitbucket.org/pypy/pypy/changeset/223ff388e219/

Log:	_curses: Implement Window.get_wch() and unget_wch().

	These two functions are needed to handle wide Unicode characters,
	like CJK and some emoji.

	(from linkmauve, last part of PR #535 and the only one that applies
	only to py3.5)

diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -403,6 +403,17 @@
             raise error("getch requires 0 or 2 arguments")
         return val
 
+    def get_wch(self, *args):
+        wch = ffi.new("int[1]")
+        if len(args) == 0:
+            val = lib.wget_wch(self._win, wch)
+        elif len(args) == 2:
+            val = lib.mvwget_wch(self._win, *args, wch)
+        else:
+            raise error("get_wch requires 0 or 2 arguments")
+        _check_ERR(val, "get_wch"):
+        return wch[0]
+
     def getkey(self, *args):
         if len(args) == 0:
             val = lib.wgetch(self._win)
@@ -1005,6 +1016,11 @@
     return _check_ERR(lib.ungetch(_chtype(ch)), "ungetch")
 
 
+def unget_wch(ch):
+    _ensure_initialised()
+    return _check_ERR(lib.unget_wch(_chtype(ch)), "unget_wch")
+
+
 def use_env(flag):
     lib.use_env(flag)
     return None
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -70,6 +70,8 @@
 typedef unsigned long... chtype;
 typedef chtype attr_t;
 
+typedef int... wint_t;
+
 typedef struct
 {
     short id;           /* ID to distinguish multiple devices */
@@ -167,6 +169,8 @@
 void filter(void);
 int flash(void);
 int flushinp(void);
+int wget_wch(WINDOW *, wint_t *);
+int mvwget_wch(WINDOW *, int, int, wint_t *);
 chtype getbkgd(WINDOW *);
 WINDOW * getwin(FILE *);
 int halfdelay(int);
@@ -242,6 +246,7 @@
 int touchwin(WINDOW *);
 int typeahead(int);
 int ungetch(int);
+int unget_wch(const wchar_t);
 int untouchwin(WINDOW *);
 void use_env(bool);
 int waddch(WINDOW *, const chtype);

From pypy.commits at gmail.com  Mon Jun 12 05:08:37 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 12 Jun 2017 02:08:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: remove vim swap files
Message-ID: <593e5a15.2899df0a.08c1.147f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: py3.5
Changeset: r91590:3cbbd9c1815e
Date: 2017-06-12 11:07 +0200
http://bitbucket.org/pypy/pypy/changeset/3cbbd9c1815e/

Log:	remove vim swap files

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
deleted file mode 100644
index 180c02ff82d3363f34a334aae22c9876d4c96481..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn b/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn
deleted file mode 100644
index 40344f6cee5cb001b73dd3a9a203015568831391..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]


From pypy.commits at gmail.com  Mon Jun 12 05:10:03 2017
From: pypy.commits at gmail.com (cfbolz)
Date: Mon, 12 Jun 2017 02:10:03 -0700 (PDT)
Subject: [pypy-commit] pypy default: add vim swn files to hgignore too
Message-ID: <593e5a6b.4ae51c0a.e93f.205f@mx.google.com>

Author: Carl Friedrich Bolz <cfbolz at gmx.de>
Branch: 
Changeset: r91591:2a8fae7d43c4
Date: 2017-06-12 11:09 +0200
http://bitbucket.org/pypy/pypy/changeset/2a8fae7d43c4/

Log:	add vim swn files to hgignore too

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,6 @@
 syntax: glob
 *.py[co]
-*.sw[po]
+*.sw[pon]
 *~
 .*.swp
 .idea

From pypy.commits at gmail.com  Mon Jun 12 10:02:56 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Jun 2017 07:02:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Document the change
Message-ID: <593e9f10.5888df0a.56975.ea33@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91593:7880a8249e05
Date: 2017-06-12 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/7880a8249e05/

Log:	Document the change

diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -6,7 +6,8 @@
 that cryptography can ship their own cffi backend)
 
 NOTE: currently, we have changed ``_cffi_src/openssl/callbacks.py`` to
-not rely on the CPython C API.
+not rely on the CPython C API, and ``_cffi_src/utils.py`` for issue #2575
+(29c9a89359e4).
 
 # Tests?
 

From pypy.commits at gmail.com  Mon Jun 12 10:02:54 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Jun 2017 07:02:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2575
Message-ID: <593e9f0e.d78adf0a.e2006.a060@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91592:29c9a89359e4
Date: 2017-06-12 16:00 +0200
http://bitbucket.org/pypy/pypy/changeset/29c9a89359e4/

Log:	Issue #2575

	Fix the current and maybe some future openssl issues. See comment.

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/utils.py b/lib_pypy/_cffi_ssl/_cffi_src/utils.py
--- a/lib_pypy/_cffi_ssl/_cffi_src/utils.py
+++ b/lib_pypy/_cffi_ssl/_cffi_src/utils.py
@@ -47,9 +47,19 @@
     # is legal, but the following will fail to compile:
     #   int foo(int);
     #   int foo(short);
+    #
+    # XXX <arigo> No, it is a bad idea.  OpenSSL itself tends to tweak
+    # the definitions, like adding a 'const' (see issue #2575).  Every
+    # time they do so, it makes a gratuitous break in this code.  It is
+    # better to rely on the C compiler for that, which is a little bit
+    # more flexible.  That's the point of set_source().  We can still
+    # re-enable the line ``#functions +`` below to get the original
+    # behavior.  (I would enable it during tests, but I don't find any
+    # custom test at all..??)
+    #
     verify_source = "\n".join(
         includes +
-        functions +
+        #functions +
         customizations
     )
     ffi = build_ffi(

From pypy.commits at gmail.com  Mon Jun 12 10:02:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Jun 2017 07:02:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: merge heads
Message-ID: <593e9f12.c687df0a.82e7b.1ad3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91594:9835812876a7
Date: 2017-06-12 16:02 +0200
http://bitbucket.org/pypy/pypy/changeset/9835812876a7/

Log:	merge heads

diff --git a/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn b/lib_pypy/_cffi_ssl/_cffi_src/.build_openssl.py.swn
deleted file mode 100644
index 180c02ff82d3363f34a334aae22c9876d4c96481..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]

diff --git a/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn b/lib_pypy/_cffi_ssl/_stdssl/.__init__.py.swn
deleted file mode 100644
index 40344f6cee5cb001b73dd3a9a203015568831391..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
GIT binary patch

[cut]


From pypy.commits at gmail.com  Mon Jun 12 12:07:23 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 12 Jun 2017 09:07:23 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: Issue #2574
Message-ID: <593ebc3b.e9afdf0a.ca8d3.bf34@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r892:1f31dd74191c
Date: 2017-06-12 18:07 +0200
http://bitbucket.org/pypy/pypy.org/changeset/1f31dd74191c/

Log:	Issue #2574

	Update the Ubuntu version range to include 16.04

diff --git a/source/download.txt b/source/download.txt
--- a/source/download.txt
+++ b/source/download.txt
@@ -82,8 +82,8 @@
 Python2.7 compatible PyPy 5.8.0
 -------------------------------
 
-* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
-* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
+* `Linux x86 binary (32bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below)
+* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below)
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Raspbian)`__ (see ``[1]`` below)
 * `ARM Hardfloat Linux binary (ARMHF/gnueabihf, tar.bz2, Ubuntu Raring)`__ (see ``[1]`` below)
 * `ARM Softfloat Linux binary (ARMEL/gnueabi, tar.bz2,  Ubuntu Precise)`__ (see ``[1]`` below)
@@ -125,7 +125,7 @@
     PyPy 2.  You are welcome to use it anyway; if you're lucky it will
     be fast in your case.  
 
-* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__ (see ``[1]`` below)
+* `Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__ (see ``[1]`` below)
 * `Source (tar.bz2)`__
 * `Source (zip)`__
 * `All our downloads,`__ including previous versions.  We also have a
@@ -155,7 +155,7 @@
 This is a special version of PyPy!  See the `Software Transactional
 Memory`_ (STM) documentation.
 
-* `PyPy-STM Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 14.04)`__
+* `PyPy-STM Linux x86-64 binary (64bit, tar.bz2 built on Ubuntu 12.04 - 16.04)`__
 
 .. _`Software Transactional Memory`: http://doc.pypy.org/en/latest/stm.html
 .. __: https://bitbucket.org/pypy/pypy/downloads/pypy-stm-2.5.1-linux64.tar.bz2

From pypy.commits at gmail.com  Mon Jun 12 16:30:31 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 12 Jun 2017 13:30:31 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Make typeslot numbers accessible
 from RPython
Message-ID: <593ef9e7.4e331c0a.9aa5.631b@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91595:1426701d43f0
Date: 2017-06-12 18:56 +0100
http://bitbucket.org/pypy/pypy/changeset/1426701d43f0/

Log:	Make typeslot numbers accessible from RPython

diff --git a/pypy/module/cpyext/include/typeslots.h b/pypy/module/cpyext/parse/typeslots.h
rename from pypy/module/cpyext/include/typeslots.h
rename to pypy/module/cpyext/parse/typeslots.h
--- a/pypy/module/cpyext/include/typeslots.h
+++ b/pypy/module/cpyext/parse/typeslots.h
@@ -1,7 +1,4 @@
 /* Do not renumber the file; these numbers are part of the stable ABI. */
-/* Disabled, see #10181 */
-#undef Py_bf_getbuffer
-#undef Py_bf_releasebuffer
 #define Py_mp_ass_subscript 3
 #define Py_mp_length 4
 #define Py_mp_subscript 5
@@ -79,7 +76,5 @@
 #define Py_am_await 77
 #define Py_am_aiter 78
 #define Py_am_anext 79
-#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000
 /* New in 3.5 */
 #define Py_tp_finalize 80
-#endif
diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py
--- a/pypy/module/cpyext/test/test_typeobject.py
+++ b/pypy/module/cpyext/test/test_typeobject.py
@@ -4,7 +4,7 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.api import generic_cpy_call
 from pypy.module.cpyext.pyobject import make_ref, from_ref
-from pypy.module.cpyext.typeobject import PyTypeObjectPtr
+from pypy.module.cpyext.typeobject import cts, PyTypeObjectPtr
 
 import sys
 import pytest
@@ -435,6 +435,9 @@
         ref = make_ref(space, w_obj)
         api.Py_DecRef(ref)
 
+    def test_typeslots(self, space):
+        assert cts.macros['Py_tp_doc'] == 56
+
 class AppTestSlots(AppTestCpythonExtensionBase):
     def setup_class(cls):
         AppTestCpythonExtensionBase.setup_class.im_func(cls)
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -41,6 +41,7 @@
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
+cts.parse_header(parse_dir / "typeslots.h")
 
 
 class W_GetSetPropertyEx(GetSetProperty):

From pypy.commits at gmail.com  Mon Jun 12 16:30:35 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 12 Jun 2017 13:30:35 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Begin implementing PyType_FromSpec()
Message-ID: <593ef9eb.84871c0a.8da60.7b19@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91597:8831a8a415be
Date: 2017-06-12 21:29 +0100
http://bitbucket.org/pypy/pypy/changeset/8831a8a415be/

Log:	Begin implementing PyType_FromSpec()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -624,6 +624,8 @@
     'PyObject_CallFinalizerFromDealloc',
     '_PyTraceMalloc_Track', '_PyTraceMalloc_Untrack',
     'PyBytes_FromFormat', 'PyBytes_FromFormatV',
+
+    'PyType_FromSpec',
 ]
 TYPES = {}
 FORWARD_DECLS = []
@@ -1348,6 +1350,7 @@
                          source_dir / "_warnings.c",
                          source_dir / "pylifecycle.c",
                          source_dir / "object.c",
+                         source_dir / "typeobject.c",
                          ]
 
 def build_eci(code, use_micronumpy=False, translating=False):
diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -113,6 +113,10 @@
 #define PyBUF_SHADOW 0x400
 /* end Py3k buffer interface */
 
+
+PyAPI_FUNC(PyObject*) PyType_FromSpec(PyType_Spec*);
+
+
 /* Flag bits for printing: */
 #define Py_PRINT_RAW    1       /* No string quotes etc. */
 
diff --git a/pypy/module/cpyext/parse/cpyext_object.h b/pypy/module/cpyext/parse/cpyext_object.h
--- a/pypy/module/cpyext/parse/cpyext_object.h
+++ b/pypy/module/cpyext/parse/cpyext_object.h
@@ -304,9 +304,10 @@
 
 typedef struct _heaptypeobject {
     PyTypeObject ht_type;
+    PyAsyncMethods as_async;
     PyNumberMethods as_number;
     PyMappingMethods as_mapping;
     PySequenceMethods as_sequence;
     PyBufferProcs as_buffer;
-    PyObject *ht_name, *ht_slots;
+    PyObject *ht_name, *ht_slots, *ht_qualname;
 } PyHeapTypeObject;
diff --git a/pypy/module/cpyext/src/typeobject.c b/pypy/module/cpyext/src/typeobject.c
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/src/typeobject.c
@@ -0,0 +1,7 @@
+#include "Python.h"
+
+PyObject *
+PyType_FromSpec(PyType_Spec *spec)
+{
+    return PyType_FromSpecWithBases(spec, NULL);
+}
diff --git a/pypy/module/cpyext/test/_testmultiphase.c b/pypy/module/cpyext/test/_testmultiphase.c
--- a/pypy/module/cpyext/test/_testmultiphase.c
+++ b/pypy/module/cpyext/test/_testmultiphase.c
@@ -6,98 +6,98 @@
 
 #include "Python.h"
 
-///* Example objects */
-//typedef struct {
-//    PyObject_HEAD
-//    PyObject            *x_attr;        /* Attributes dictionary */
-//} ExampleObject;
-//
-///* Example methods */
-//
-//static int
-//Example_traverse(ExampleObject *self, visitproc visit, void *arg)
-//{
-//    Py_VISIT(self->x_attr);
-//    return 0;
-//}
-//
-//static int
-//Example_finalize(ExampleObject *self)
-//{
-//    Py_CLEAR(self->x_attr);
-//    return 0;
-//}
-//
-//static PyObject *
-//Example_demo(ExampleObject *self, PyObject *args)
-//{
-//    PyObject *o = NULL;
-//    if (!PyArg_ParseTuple(args, "|O:demo", &o))
-//        return NULL;
-//    if (o != NULL && PyUnicode_Check(o)) {
-//        Py_INCREF(o);
-//        return o;
-//    }
-//    Py_INCREF(Py_None);
-//    return Py_None;
-//}
-//
-//
-//static PyMethodDef Example_methods[] = {
-//    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
-//        PyDoc_STR("demo() -> None")},
-//    {NULL,              NULL}           /* sentinel */
-//};
-//
-//static PyObject *
-//Example_getattro(ExampleObject *self, PyObject *name)
-//{
-//    if (self->x_attr != NULL) {
-//        PyObject *v = PyDict_GetItem(self->x_attr, name);
-//        if (v != NULL) {
-//            Py_INCREF(v);
-//            return v;
-//        }
-//    }
-//    return PyObject_GenericGetAttr((PyObject *)self, name);
-//}
-//
-//static int
-//Example_setattr(ExampleObject *self, char *name, PyObject *v)
-//{
-//    if (self->x_attr == NULL) {
-//        self->x_attr = PyDict_New();
-//        if (self->x_attr == NULL)
-//            return -1;
-//    }
-//    if (v == NULL) {
-//        int rv = PyDict_DelItemString(self->x_attr, name);
-//        if (rv < 0)
-//            PyErr_SetString(PyExc_AttributeError,
-//                "delete non-existing Example attribute");
-//        return rv;
-//    }
-//    else
-//        return PyDict_SetItemString(self->x_attr, name, v);
-//}
-//
-//static PyType_Slot Example_Type_slots[] = {
-//    {Py_tp_doc, "The Example type"},
-//    {Py_tp_finalize, Example_finalize},
-//    {Py_tp_traverse, Example_traverse},
-//    {Py_tp_getattro, Example_getattro},
-//    {Py_tp_setattr, Example_setattr},
-//    {Py_tp_methods, Example_methods},
-//    {0, 0},
-//};
-//
-//static PyType_Spec Example_Type_spec = {
-//    "_testimportexec.Example",
-//    sizeof(ExampleObject),
-//    0,
-//    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
-//    Example_Type_slots
-//};
+/* Example objects */
+typedef struct {
+    PyObject_HEAD
+    PyObject            *x_attr;        /* Attributes dictionary */
+} ExampleObject;
+
+/* Example methods */
+
+static int
+Example_traverse(ExampleObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(self->x_attr);
+    return 0;
+}
+
+static int
+Example_finalize(ExampleObject *self)
+{
+    Py_CLEAR(self->x_attr);
+    return 0;
+}
+
+static PyObject *
+Example_demo(ExampleObject *self, PyObject *args)
+{
+    PyObject *o = NULL;
+    if (!PyArg_ParseTuple(args, "|O:demo", &o))
+        return NULL;
+    if (o != NULL && PyUnicode_Check(o)) {
+        Py_INCREF(o);
+        return o;
+    }
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
+
+static PyMethodDef Example_methods[] = {
+    {"demo",            (PyCFunction)Example_demo,  METH_VARARGS,
+        PyDoc_STR("demo() -> None")},
+    {NULL,              NULL}           /* sentinel */
+};
+
+static PyObject *
+Example_getattro(ExampleObject *self, PyObject *name)
+{
+    if (self->x_attr != NULL) {
+        PyObject *v = PyDict_GetItem(self->x_attr, name);
+        if (v != NULL) {
+            Py_INCREF(v);
+            return v;
+        }
+    }
+    return PyObject_GenericGetAttr((PyObject *)self, name);
+}
+
+static int
+Example_setattr(ExampleObject *self, char *name, PyObject *v)
+{
+    if (self->x_attr == NULL) {
+        self->x_attr = PyDict_New();
+        if (self->x_attr == NULL)
+            return -1;
+    }
+    if (v == NULL) {
+        int rv = PyDict_DelItemString(self->x_attr, name);
+        if (rv < 0)
+            PyErr_SetString(PyExc_AttributeError,
+                "delete non-existing Example attribute");
+        return rv;
+    }
+    else
+        return PyDict_SetItemString(self->x_attr, name, v);
+}
+
+static PyType_Slot Example_Type_slots[] = {
+    {Py_tp_doc, "The Example type"},
+    {Py_tp_finalize, Example_finalize},
+    {Py_tp_traverse, Example_traverse},
+    {Py_tp_getattro, Example_getattro},
+    {Py_tp_setattr, Example_setattr},
+    {Py_tp_methods, Example_methods},
+    {0, 0},
+};
+
+static PyType_Spec Example_Type_spec = {
+    "_testimportexec.Example",
+    sizeof(ExampleObject),
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE,
+    Example_Type_slots
+};
 
 /* Function of two integers returning integer */
 
@@ -158,18 +158,18 @@
 }
 */
 
-//static PyType_Slot Str_Type_slots[] = {
-//    {Py_tp_base, NULL}, /* filled out in module exec function */
-//    {0, 0},
-//};
-//
-//static PyType_Spec Str_Type_spec = {
-//    "_testimportexec.Str",
-//    0,
-//    0,
-//    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
-//    Str_Type_slots
-//};
+static PyType_Slot Str_Type_slots[] = {
+    {Py_tp_base, NULL}, /* filled out in module exec function */
+    {0, 0},
+};
+
+static PyType_Spec Str_Type_spec = {
+    "_testimportexec.Str",
+    0,
+    0,
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    Str_Type_slots
+};
 
 static PyMethodDef testexport_methods[] = {
     {"foo",             testexport_foo,         METH_VARARGS,
@@ -179,46 +179,46 @@
     {NULL,              NULL}           /* sentinel */
 };
 
-//static int execfunc(PyObject *m)
-//{
-//    PyObject *temp = NULL;
-//
-//    /* Due to cross platform compiler issues the slots must be filled
-//     * here. It's required for portability to Windows without requiring
-//     * C++. */
-//    Str_Type_slots[0].pfunc = &PyUnicode_Type;
-//
-//    /* Add a custom type */
-//    temp = PyType_FromSpec(&Example_Type_spec);
-//    if (temp == NULL)
-//        goto fail;
-//    if (PyModule_AddObject(m, "Example", temp) != 0)
-//        goto fail;
-//
-//    /* Add an exception type */
-//    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
-//    if (temp == NULL)
-//        goto fail;
-//    if (PyModule_AddObject(m, "error", temp) != 0)
-//        goto fail;
-//
-//    /* Add Str */
-//    temp = PyType_FromSpec(&Str_Type_spec);
-//    if (temp == NULL)
-//        goto fail;
-//    if (PyModule_AddObject(m, "Str", temp) != 0)
-//        goto fail;
-//
-//    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
-//        goto fail;
-//
-//    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
-//        goto fail;
-//
-//    return 0;
-// fail:
-//    return -1;
-//}
+static int execfunc(PyObject *m)
+{
+    PyObject *temp = NULL;
+
+    /* Due to cross platform compiler issues the slots must be filled
+     * here. It's required for portability to Windows without requiring
+     * C++. */
+    Str_Type_slots[0].pfunc = &PyUnicode_Type;
+
+    /* Add a custom type */
+    temp = PyType_FromSpec(&Example_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Example", temp) != 0)
+        goto fail;
+
+    /* Add an exception type */
+    temp = PyErr_NewException("_testimportexec.error", NULL, NULL);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "error", temp) != 0)
+        goto fail;
+
+    /* Add Str */
+    temp = PyType_FromSpec(&Str_Type_spec);
+    if (temp == NULL)
+        goto fail;
+    if (PyModule_AddObject(m, "Str", temp) != 0)
+        goto fail;
+
+    if (PyModule_AddIntConstant(m, "int_const", 1969) != 0)
+        goto fail;
+
+    if (PyModule_AddStringConstant(m, "str_const", "something different") != 0)
+        goto fail;
+
+    return 0;
+ fail:
+    return -1;
+}
 
 /* Helper for module definitions; there'll be a lot of them */
 #define TEST_MODULE_DEF(name, slots, methods) { \
@@ -234,7 +234,7 @@
 }
 
 PyModuleDef_Slot main_slots[] = {
-    //{Py_mod_exec, execfunc},
+    {Py_mod_exec, execfunc},
     {0, NULL},
 };
 
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -145,3 +145,17 @@
         raises(AttributeError, 'module.__path__')
         assert module is sys.modules[NAME]
         assert isinstance(module.__loader__, machinery.ExtensionFileLoader)
+
+    def test_functionality(self):
+        import types
+        NAME = '_testmultiphase'
+        module = self.import_module(name=NAME)
+        assert isinstance(module, types.ModuleType)
+        ex = module.Example()
+        assert ex.demo('abcd') == 'abcd'
+        assert ex.demo() is None
+        raises(AttributeError, ex.abc)
+        ex.abc = 0
+        assert ex.abc == 0
+        assert module.foo(9, 9) == 18
+
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -14,7 +14,7 @@
     cpython_api, cpython_struct, bootstrap_function, Py_ssize_t, Py_ssize_tP,
     slot_function, generic_cpy_call, Py_TPFLAGS_READY, Py_TPFLAGS_READYING, Py_buffer,
     Py_TPFLAGS_HEAPTYPE, METH_VARARGS, METH_KEYWORDS, CANNOT_FAIL,
-    build_type_checkers,
+    build_type_checkers, Py_TPFLAGS_BASETYPE,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     cts, parse_dir)
 from pypy.module.cpyext.cparser import parse_source
@@ -24,7 +24,7 @@
 from pypy.module.cpyext.modsupport import convert_method_defs
 from pypy.module.cpyext.pyobject import (
     PyObject, make_ref, from_ref, get_typedescr, make_typedescr,
-    track_reference, Py_DecRef, as_pyobj)
+    track_reference, Py_DecRef, as_pyobj, incref)
 from pypy.module.cpyext.slotdefs import (
     slotdefs_for_tp_slots, slotdefs_for_wrappers, get_slot_tp_function,
     llslot)
@@ -867,6 +867,78 @@
     return generic_cpy_call(
         space, type.c_tp_alloc, type, 0)
 
+ at cts.decl("""PyObject *
+    PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)""",
+    result_is_ll=True)
+def PyType_FromSpecWithBases(space, spec, bases):
+    res = PyType_GenericAlloc(space, space.w_type, 0)
+    res = cts.cast('PyHeapTypeObject *', res)
+    typ = res.c_ht_type
+    typ.c_tp_flags = rffi.cast(lltype.Signed, spec.c_flags)
+    typ.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
+    #s = 'foo'
+    #res.c_ht_name = PyUnicode_FromString(s)
+    #res.c_ht_qualname = res.c_ht_name
+    incref(space, res.c_ht_qualname)
+    typ.c_tp_name = spec.c_name
+    slotdefs = rffi.cast(rffi.CArrayPtr(cts.gettype('PyType_Slot')), spec.c_slots)
+    if not bases:
+        w_base = space.w_object
+        bases_w = []
+        i = 0
+        while True:
+            slotdef = slotdefs[i]
+            if slotdef.c_slot == 0:
+                break
+            if slotdef.c_slot == cts.macros['Py_tp_base']:
+                w_base = from_ref(space, cts.cast('PyObject*', slotdef.c_pfunc))
+            elif slotdef.c_slot == cts.macros['Py_tp_bases']:
+                bases = cts.cast('PyObject*', slotdef.c_pfunc)
+                bases_w = space.fixedview(from_ref(space, bases))
+            i += 1
+        if not bases_w:
+            bases_w = [w_base]
+    else:
+        bases_w = space.fixed_view(from_ref(space, bases))
+    w_base = best_base(space, bases_w)
+    base = cts.cast('PyTypeObject*', make_ref(space, w_base))
+    if False:  # not base.c_tp_flags & Py_TPFLAGS_BASETYPE:
+        raise oefmt(space.w_TypeError,
+            "type '%s' is not an acceptable base type",
+            rffi.charp2str(base.c_tp_name))
+
+    typ.c_tp_as_async = res.c_as_async
+    typ.c_tp_as_number = res.c_as_number
+    typ.c_tp_as_sequence = res.c_as_sequence
+    typ.c_tp_as_mapping = res.c_as_mapping
+    typ.c_tp_as_buffer = res.c_as_buffer
+    typ.c_tp_bases = bases
+    typ.c_tp_base = base
+    typ.c_tp_basicsize = cts.cast('Py_ssize_t', spec.c_basicsize)
+    typ.c_tp_itemsize = cts.cast('Py_ssize_t', spec.c_itemsize)
+
+    i = 0
+    while True:
+        slotdef = slotdefs[i]
+        if slotdef.c_slot == 0:
+            break
+        slot = slotdef.c_slot
+        if slot < 0:  # or slot > len(slotoffsets):
+            raise oefmt(space.w_RuntimeError, "invalid slot offset")
+        if slot in (cts.macros['Py_tp_base'], cts.macros['Py_tp_bases']):
+            # Processed above
+            i += 1
+            continue
+        #fill_slot(res, slot, slotdef.c_pfunc)
+        # XXX: need to make a copy of the docstring slot, which usually
+        # points to a static string literal
+        i += 1
+
+    if not typ.c_tp_dealloc:
+        typ.c_tp_dealloc = llslot(space, subtype_dealloc)
+    py_type_ready(space, typ)
+    return cts.cast('PyObject*', res)
+
 @cpython_api([PyTypeObjectPtr, PyObject], PyObject, error=CANNOT_FAIL,
              result_borrowed=True)
 def _PyType_Lookup(space, type, w_name):
@@ -894,4 +966,3 @@
         return
     if w_obj.is_cpytype():
         w_obj.mutated(None)
-

From pypy.commits at gmail.com  Mon Jun 12 16:30:33 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 12 Jun 2017 13:30:33 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Add result_is_ll option to cts.decl()
Message-ID: <593ef9e9.91d81c0a.6b9f2.8f84@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91596:527c72b5b5d9
Date: 2017-06-12 21:23 +0100
http://bitbucket.org/pypy/pypy/changeset/527c72b5b5d9/

Log:	Add result_is_ll option to cts.decl()

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -487,13 +487,15 @@
     return decorate
 
 def api_func_from_cdef(func, cdef, cts,
-        error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+        error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+        result_is_ll=False):
     func._always_inline_ = 'try'
     cdecl = cts.parse_func(cdef)
     RESULT = cdecl.get_llresult(cts)
     api_function = ApiFunction(
         cdecl.get_llargs(cts), RESULT, func,
-        error=_compute_error(error, RESULT), cdecl=cdecl)
+        error=_compute_error(error, RESULT), cdecl=cdecl,
+        result_is_ll=result_is_ll)
     FUNCTIONS_BY_HEADER[header][cdecl.name] = api_function
     unwrapper = api_function.get_unwrapper()
     unwrapper.func = func
@@ -698,10 +700,12 @@
 
 
 class CpyextTypeSpace(CTypeSpace):
-    def decl(self, cdef, error=_NOT_SPECIFIED, header=DEFAULT_HEADER):
+    def decl(self, cdef, error=_NOT_SPECIFIED, header=DEFAULT_HEADER,
+            result_is_ll=False):
         def decorate(func):
             return api_func_from_cdef(
-                    func, cdef, self, error=error, header=header)
+                func, cdef, self, error=error, header=header,
+                result_is_ll=result_is_ll)
         return decorate
 
 

From pypy.commits at gmail.com  Tue Jun 13 03:52:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Jun 2017 00:52:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Obscuuure.  On Ubuntu specifically,
 for ncurses on pypy3 we need:
Message-ID: <593f99a7.0d931c0a.a53a1.1a6b@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91598:518cd1383456
Date: 2017-06-13 09:51 +0200
http://bitbucket.org/pypy/pypy/changeset/518cd1383456/

Log:	Obscuuure. On Ubuntu specifically, for ncurses on pypy3 we need:

	(1) apt-get install libncursesw-dev

	(2) add an include dir, otherwise it picks up another different
	ncurses.h

diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -1,3 +1,4 @@
+import os
 from cffi import FFI, VerificationError
 
 
@@ -17,6 +18,11 @@
     # error message
     raise e_last
 
+def find_curses_include_dirs():
+    if os.path.exists('/usr/include/ncursesw'):
+        return ['/usr/include/ncursesw']
+    return []
+
 
 ffi = FFI()
 
@@ -59,7 +65,8 @@
 void _m_getsyx(int *yx) {
     getsyx(yx[0], yx[1]);
 }
-""", libraries=[find_curses_library(), 'panel'])
+""", libraries=[find_curses_library(), 'panel'],
+     include_dirs=find_curses_include_dirs())
 
 
 ffi.cdef("""

From pypy.commits at gmail.com  Tue Jun 13 03:57:02 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Jun 2017 00:57:02 -0700 (PDT)
Subject: [pypy-commit] pypy default: This package is needed on Ubuntu for
 PyPy3
Message-ID: <593f9ace.53061c0a.d43ef.0dec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91599:c290c73933b9
Date: 2017-06-13 09:56 +0200
http://bitbucket.org/pypy/pypy/changeset/c290c73933b9/

Log:	This package is needed on Ubuntu for PyPy3

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -93,7 +93,8 @@
     libsqlite3
 
 curses
-    libncurses
+    libncurses-dev   (for PyPy2)
+    libncursesw-dev  (for PyPy3)
 
 gdbm
     libgdbm-dev
@@ -111,7 +112,7 @@
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
     tk-dev libgc-dev python-cffi \
-    liblzma-dev  # For lzma on PyPy3.
+    liblzma-dev libncursesw-dev      # these two only needed on PyPy3
 
 On Fedora::
 

From pypy.commits at gmail.com  Tue Jun 13 03:57:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Jun 2017 00:57:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Name "Ubuntu" in addition to "Debian"
Message-ID: <593f9af7.2196df0a.a64d.593f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91600:4aebe6f15c48
Date: 2017-06-13 09:57 +0200
http://bitbucket.org/pypy/pypy/changeset/4aebe6f15c48/

Log:	Name "Ubuntu" in addition to "Debian"

diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -107,7 +107,8 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian, this is the command to install all build-time dependencies::
+On Debian and Ubuntu, this is the command to install all build-time
+dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \

From pypy.commits at gmail.com  Tue Jun 13 04:32:03 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 13 Jun 2017 01:32:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix some tests
Message-ID: <593fa303.83b2df0a.15884.493c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91601:609a3cdf9cf7
Date: 2017-06-13 10:20 +0200
http://bitbucket.org/pypy/pypy/changeset/609a3cdf9cf7/

Log:	Fix some tests

diff --git a/pypy/module/pypyjit/test_pypy_c/model.py b/pypy/module/pypyjit/test_pypy_c/model.py
--- a/pypy/module/pypyjit/test_pypy_c/model.py
+++ b/pypy/module/pypyjit/test_pypy_c/model.py
@@ -253,6 +253,7 @@
 
 
 class OpMatcher(object):
+    DUMMY = 1000
 
     def __init__(self, ops, id=None):
         self.ops = ops
@@ -283,6 +284,13 @@
         #
         if line.strip() == 'guard_not_invalidated?':
             return 'guard_not_invalidated', None, [], '...', False
+        if line.strip() == 'dummy_get_utf8?':
+            cls.DUMMY += 1
+            return ('getfield_gc_r',
+                    'p%d' % cls.DUMMY,
+                    ['ConstPtr(ptr%d)' % cls.DUMMY],
+                    '<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>',
+                    False)
         # find the resvar, if any
         if ' = ' in line:
             resvar, _, line = line.partition(' = ')
diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py
--- a/pypy/module/pypyjit/test_pypy_c/test_call.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_call.py
@@ -81,7 +81,7 @@
         assert log.opnames(ops) == ["getfield_gc_r"]
         #
         assert entry_bridge.match_by_id('call', """
-            p30 = getfield_gc_r(ConstPtr(ptr29), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+            dummy_get_utf8?
             p38 = call_r(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=<Callr . i EF=1 OS=5>)
             p99 = getfield_gc_r(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_sys_exc_operror .*>)
             p39 = getfield_gc_r(p38, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>)
@@ -443,7 +443,7 @@
             i22 = getfield_gc_i(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>)
             i24 = int_lt(i22, 5000)
             guard_true(i24, descr=...)
-            p21 = getfield_gc_r(ConstPtr(ptr20), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+            dummy_get_utf8?
             guard_not_invalidated(descr=...)
             p29 = call_r(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=<Callr . i EF=1 OS=5>)
             p99 = getfield_gc_r(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_sys_exc_operror .*>)
diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py
--- a/pypy/module/pypyjit/test_pypy_c/test_misc.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py
@@ -321,7 +321,7 @@
         loop, = log.loops_by_id("globalread", is_entry_bridge=True)
         assert loop.match_by_id("globalread", """
             # only a dead read
-            p26 = getfield_gc_r(ConstPtr(ptr25), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
+            dummy_get_utf8?
         """)
 
     def test_eval(self):
diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -57,7 +57,6 @@
         assert loop.match("""
             i88 = int_lt(i83, i36)
             guard_true(i88, descr=...)
-            p90 = getfield_gc_r(ConstPtr(ptr89), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
             i92 = int_eq(i83, %d)
             i94 = call_i(ConstClass(ll_int_py_mod__Signed_Signed), i83, i46, descr=<Calli . ii EF=0 OS=14>)
@@ -66,7 +65,6 @@
             i97 = int_ge(i94, i53)
             guard_false(i97, descr=...)
             i98 = strgetitem(p52, i94)
-            p100 = getfield_gc_r(ConstPtr(ptr99), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             p101 = force_token()
             p103 = newstr(1)
             strsetitem(p103, 0, i98)
@@ -83,7 +81,6 @@
             p116 = call_r(ConstClass(ll_build_trampoline__), p104, descr=<Callr . r EF=5>)
             guard_no_exception(descr=...)
             guard_nonnull(p116, descr=...)
-            p118 = getfield_gc_r(ConstPtr(ptr117), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
             i119 = int_ge(i94, i46)
             guard_false(i119, descr=...)
@@ -113,9 +110,7 @@
         assert loop.match("""
             i11 = int_lt(i6, i7)
             guard_true(i11, descr=...)
-            p70 = getfield_gc_r(ConstPtr(ptr69), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             guard_not_invalidated(descr=...)
-            p72 = getfield_gc_r(ConstPtr(ptr71), descr=<FieldP pypy.objspace.std.unicodeobject.W_UnicodeObject.inst__utf8 .>)
             i13 = int_eq(i6, %d)         # value provided below
 
             # "mod 10" block:
diff --git a/pypy/module/pypyjit/test_pypy_c/test_struct.py b/pypy/module/pypyjit/test_pypy_c/test_struct.py
--- a/pypy/module/pypyjit/test_pypy_c/test_struct.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_struct.py
@@ -34,6 +34,7 @@
             # on little endian machines, we take the fast path and store the
             # value using gc_store_indexed
             assert loop.match_by_id("pack", """
+                dummy_get_utf8?
                 guard_not_invalidated(descr=...)
                 # struct.pack
                 %s
@@ -42,6 +43,7 @@
             """ % extra)
         else:
             assert loop.match_by_id("pack", """
+                dummy_get_utf8?
                 guard_not_invalidated(descr=...)
                 # struct.pack
                 %s
@@ -56,6 +58,8 @@
 
         if sys.byteorder == 'little':
             assert loop.match_by_id("unpack", """
+                dummy_get_utf8?
+                dummy_get_utf8?
                 # struct.unpack
                 i91 = gc_load_indexed_i(p88, 0, 1, _, -4)
             """)
@@ -64,6 +68,8 @@
             # a char buffer and then use load gc to read the integer,
             # here manual shifting is applied
             assert loop.match_by_id("unpack", """
+                dummy_get_utf8?
+                dummy_get_utf8?
                 # struct.unpack
                 i95 = int_lshift(i90, 8)
                 i96 = int_or(i88, i95)
@@ -92,6 +98,7 @@
         if sys.byteorder == 'little':
             loop, = log.loops_by_filename(self.filepath)
             assert loop.match_by_id('pack', """
+                dummy_get_utf8?
                 guard_not_invalidated(descr=...)
                 # struct.pack
                 p85 = newstr(8)
@@ -101,6 +108,7 @@
             """ % extra)
 
             assert loop.match_by_id('unpack', """
+                dummy_get_utf8?
                 # struct.unpack
                 i90 = gc_load_indexed_i(p88, 0, 1, _, -4)
                 i91 = gc_load_indexed_i(p88, 4, 1, _, -4)
@@ -183,7 +191,9 @@
         assert log.result == main(1000)
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match_by_id('pack_into', """\
+            dummy_get_utf8?
             guard_not_invalidated(descr=...)
+            dummy_get_utf8?
             p68 = getfield_gc_r(p14, descr=<FieldP pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__data \d+>)
             i69 = getfield_gc_i(p68, descr=<FieldS list.length \d+>)
             i70 = getfield_gc_i(p14, descr=<FieldS pypy.objspace.std.bytearrayobject.W_BytearrayObject.inst__offset \d+>)

From pypy.commits at gmail.com  Wed Jun 14 04:00:01 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 14 Jun 2017 01:00:01 -0700 (PDT)
Subject: [pypy-commit] pypy stmgc-c8: (tobweber) fix wrong thread state
 after waiting for free segment
Message-ID: <5940ed01.158bdf0a.63108.13df@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c8
Changeset: r91602:8d2c43276bbe
Date: 2017-06-14 09:48 +0200
http://bitbucket.org/pypy/pypy/changeset/8d2c43276bbe/

Log:	(tobweber) fix wrong thread state after waiting for free segment

diff --git a/pypy/stm/print_stm_log.py b/pypy/stm/print_stm_log.py
--- a/pypy/stm/print_stm_log.py
+++ b/pypy/stm/print_stm_log.py
@@ -118,6 +118,7 @@
         self._transaction_aborting = False
         self._transaction_inev = None
         self._transaction_detached_time = 0.0
+        self._transaction_awaiting_free_seg = False
         self._in_minor_coll = None
         assert self._prev[1] == "stop"
 
@@ -169,6 +170,10 @@
         if self._transaction_inev is None:
             self._transaction_inev = [entry, None]
 
+    def transaction_await_free_segment(self, entry):
+        self._transaction_awaiting_free_seg = True
+        self.transaction_pause(entry)
+
     def transaction_pause(self, entry):
         self.progress(entry.timestamp, "pause")
         if (entry.event == STM_WAIT_OTHER_INEVITABLE and
@@ -176,7 +181,12 @@
             self._transaction_inev[1] = entry.timestamp
 
     def transaction_unpause(self, entry, out_conflicts):
-        self.progress(entry.timestamp, "run")
+        if self._transaction_awaiting_free_seg:
+            # waiting for free segment does not mean we are running now
+            self.progress(entry.timestamp, "stop")
+            self._transaction_awaiting_free_seg = False
+        else:
+            self.progress(entry.timestamp, "run")
         if self._transaction_inev and self._transaction_inev[1] is not None:
             wait_time = entry.timestamp - self._transaction_inev[1]
             self.wait_for_other_inev(wait_time, out_conflicts)
@@ -310,8 +320,9 @@
             t.become_inevitable(entry)
         elif entry.event == STM_CONTENTION_WRITE_READ:
             t.contention_write_read(entry, conflicts)
-        elif entry.event in (STM_WAIT_FREE_SEGMENT,
-                             STM_WAIT_SYNCING,
+        elif entry.event == STM_WAIT_FREE_SEGMENT:
+            t.transaction_await_free_segment(entry)
+        elif entry.event in (STM_WAIT_SYNCING,
                              STM_WAIT_SYNC_PAUSE,
                              STM_WAIT_OTHER_INEVITABLE):
             t.transaction_pause(entry)

From pypy.commits at gmail.com  Wed Jun 14 04:00:04 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 14 Jun 2017 01:00:04 -0700 (PDT)
Subject: [pypy-commit] pypy stmgc-c8: add passing stmrewrite test
Message-ID: <5940ed04.239ddf0a.3bbf2.17f5@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: stmgc-c8
Changeset: r91603:333cd7c29400
Date: 2017-06-14 09:51 +0200
http://bitbucket.org/pypy/pypy/changeset/333cd7c29400/

Log:	add passing stmrewrite test

diff --git a/rpython/jit/backend/llsupport/test/test_stmrewrite.py b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
--- a/rpython/jit/backend/llsupport/test/test_stmrewrite.py
+++ b/rpython/jit/backend/llsupport/test/test_stmrewrite.py
@@ -356,7 +356,7 @@
             setfield_gc(p1, p2, descr=tzdescr)
             label(p1, i3)
             setfield_gc(p1, i3, descr=tydescr)
-            jump(p1)
+            jump(p1, i3)
         """, """
             [p1, p2, i3]
             cond_call_gc_wb(p1, descr=wbdescr)
@@ -364,7 +364,24 @@
             label(p1, i3)
             cond_call_gc_wb(p1, descr=wbdescr)
             gc_store(p1, 0,  i3, %(tydescr.field_size)s)
-            jump(p1)
+            jump(p1, i3)
+        """)
+
+    def test_rewrite_getfield_over_label(self):
+        self.check_rewrite("""
+            [p1, p2, i3]
+            p3 = getfield_gc_r(p1, descr=tzdescr)
+            label(p1, i3)
+            p4 = getfield_gc_r(p1, descr=tzdescr)
+            jump(p4, i3)
+        """, """
+            [p1, p2, i3]
+            stm_read(p1)
+            p3 = gc_load_r(p1,  %(tzdescr.field_size)s, %(tzdescr.field_size)s)
+            label(p1, i3)
+            stm_read(p1)
+            p4 = gc_load_r(p1,  %(tzdescr.field_size)s, %(tzdescr.field_size)s)
+            jump(p4, i3)
         """)
 
     def test_remove_debug_merge_point(self):
@@ -583,7 +600,7 @@
             i3 = getfield_raw_i(i1, descr=tydescr)
             label(i1, i2, i3)
             i4 = getfield_raw_i(i2, descr=tydescr)
-            jump(i3, i4)
+            jump(i3, i4, i4)
         """, """
             [i1, i2]
             $INEV
@@ -591,7 +608,7 @@
             label(i1, i2, i3)
             $INEV
             i4 = gc_load_i(i2, 0, -%(tydescr.field_size)d)
-            jump(i3, i4)
+            jump(i3, i4, i4)
         """)
 
     def test_getarrayitem_raw(self):

From pypy.commits at gmail.com  Wed Jun 14 07:38:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 14 Jun 2017 04:38:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: Avoid one instruction in the endless
 stream of header/footer instructions
Message-ID: <59412020.089c1c0a.50c92.73cf@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91604:9a5adcd4d3ff
Date: 2017-06-14 13:37 +0200
http://bitbucket.org/pypy/pypy/changeset/9a5adcd4d3ff/

Log:	Avoid one instruction in the endless stream of header/footer
	instructions

diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -980,7 +980,8 @@
         from rpython.rlib.rvmprof.rvmprof import cintf
         # edx = address of pypy_threadlocal_s
         self.mc.MOV_rs(edx.value, THREADLOCAL_OFS)
-        self.mc.AND_ri(edx.value, ~1)
+        if self._is_asmgcc():
+            self.mc.AND_ri(edx.value, ~1)
         # eax = (our local vmprof_tl_stack).next
         self.mc.MOV_rs(eax.value, (FRAME_FIXED_SIZE - 4 + 0) * WORD)
         # save in vmprof_tl_stack the value eax

From pypy.commits at gmail.com  Wed Jun 14 09:33:18 2017
From: pypy.commits at gmail.com (Raemi)
Date: Wed, 14 Jun 2017 06:33:18 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-reshare-pages: understand and document what
 this branch is doing
Message-ID: <59413b1e.94da1c0a.3acf3.0028@mx.google.com>

Author: Remi Meier <remi.meier at gmail.com>
Branch: c8-reshare-pages
Changeset: r2073:e08f018e54b8
Date: 2017-06-14 15:32 +0200
http://bitbucket.org/pypy/stmgc/changeset/e08f018e54b8/

Log:	understand and document what this branch is doing

	A try to write down the logic of the page resharing approach while
	also improving the code.

diff --git a/c8/doc/page-resharing.md b/c8/doc/page-resharing.md
new file mode 100644
--- /dev/null
+++ b/c8/doc/page-resharing.md
@@ -0,0 +1,79 @@
+
+
+# Page Resharing #
+
+The idea of page resharing is that, after a while of running a multi-threaded
+program, all `S` segments have read most pages of the heap. Thus, nearly all of
+the heap memory is duplicated `S` times. While very wasteful, such duplication
+also negatively affects the performance of the validation operation. During
+validation, all committed changes need to be imported for all accessible pages
+in a segment. Thus, if all pages are accessible, all changes need to be copied
+in. Instead, it is probably better to sometimes mark pages inaccessible again if
+they are rarely accessed in that segment, and we therefore don't need to import
+changes for these pages; lessening the work done by the validation operation.
+
+
+## Situation without Resharing ##
+
+Pages can only be ACCESSIBLE (readable *and* writable) or INACCESSIBLE (neither)
+in a segment. Seg0's pages are always ACCESSIBLE.
+
+New (old) objects get allocated during minor GCs in pages of the current
+segment. On commit, these objects get copied to seg0 and to all segments in
+which that page is also ACCESSIBLE. 
+
+Whenever we access an obj (reading *or* writing) and the obj's page is not
+ACCESSIBLE yet, we get a signal and transition the page from INACCESSIBLE to
+ACCESSIBLE. Other segments are unaffected.
+
+
+## Situation with Resharing ##
+
+Pages can be NOACC, RO, ACC. 
+
+`RO` provides the revision of seg0. Whenever a TX makes a page `ACC` while there
+are `RO` pages around, make all `RO` pages `NOACC` in all segments. This eager
+approach means that we do not need all privatization-locks (write) in many
+places where we would otherwise need a `RO->NOACC` transition. Also, we expect
+the `RO` pages to go out-of-date when we commit anyway (soon).
+
+*INVARIANT*: whenever a segment has a `RO` page, that page has the same content
+as seg0's version of that page.
+
+*INVARIANT*: if there are `RO` pages around, no segment has the page `ACC`
+
+*PROPERTY I*: Once there page is `ACC`, it stays `ACC` until major GC; no `RO` can
+exist until major GC reshares (`ACC -> RO`) and makes seg0 up-to-date.
+
+*PROPERTY II*: Validation will never try to import into `RO` pages, since (I)
+guarantees that it wouldn't be a `RO` anymore if there was a change to import.
+
+
+In signal handler:
+
+    if read or write:
+      if is `RO`:
+        `RO -> ACC` (and `RO -> NOACC` for all others)
+      else if is `NOACC`:
+        if !is_write and noone has `ACC`:
+          `NOACC -> RO`
+        else:
+          `NOACC -> ACC`
+
+On validate: always imports into `ACC`, into `RO` would be a bug.
+
+During major GC:
+
+ 1. Validation of seg0: gets all changes; any `RO` views still around means that
+    there was *no change* in those pages, so the views stay valid.
+ 2. All other segments validate their `ACC` pages; again `RO` pages *cannot*
+    have changes that need importing.
+ 3. While tracing modified objs and overflow objs, remember pages with
+    modifications. These *must not* change from `ACC` to `RO`.
+ 4. Loop over some pages (resharing can be distributed over several major GCs),
+    and do `ACC -> RO` for all segments iff the previous step shows that to be
+    valid. After that, the INVARIANTs need to hold.
+ 
+
+
+
diff --git a/c8/stm/core.c b/c8/stm/core.c
--- a/c8/stm/core.c
+++ b/c8/stm/core.c
@@ -67,8 +67,10 @@
         /* never import anything into READONLY pages */
         assert(get_page_status_in(my_segnum, current_page_num) != PAGE_READONLY);
 
-        if (pagenum == -1) {
-            if (get_page_status_in(my_segnum, current_page_num) != PAGE_ACCESSIBLE)
+        if (pagenum == -1UL) {
+            assert(IMPLY(my_segnum == 0,
+                         get_page_status_in(my_segnum, current_page_num) == PAGE_ACCESSIBLE));
+            if (get_page_status_in(my_segnum, current_page_num) == PAGE_NO_ACCESS)
                 continue;
         } else if (pagenum != current_page_num) {
             continue;
@@ -793,7 +795,7 @@
 }
 
 
-static void touch_all_pages_of_obj(object_t *obj, size_t obj_size)
+static void make_all_pages_of_obj_accessible(object_t *obj, size_t obj_size)
 {
     /* XXX: make this function not needed */
     int my_segnum = STM_SEGMENT->segment_num;
@@ -806,7 +808,7 @@
         end_page = (((uintptr_t)obj) + obj_size - 1) / 4096UL;
     }
 
-    dprintf(("touch_all_pages_of_obj(%p, %lu): %ld-%ld\n",
+    dprintf(("make_all_pages_of_obj_accessible(%p, %lu): %ld-%ld\n",
              obj, obj_size, first_page, end_page));
 
     acquire_privatization_lock(STM_SEGMENT->segment_num);
@@ -816,8 +818,8 @@
 
         if (get_page_status_in(my_segnum, page) != PAGE_ACCESSIBLE) {
             release_privatization_lock(STM_SEGMENT->segment_num);
-            /* emulate pagefault -> PAGE_ACCESSIBLE/READONLY: */
-            handle_segfault_in_page(page, false);
+            /* emulate pagefault -> PAGE_ACCESSIBLE: */
+            handle_segfault_in_page(page, true);
             volatile char *dummy = REAL_ADDRESS(STM_SEGMENT->segment_base, page * 4096UL);
             *dummy = *dummy;            /* force segfault (incl. writing) */
             acquire_privatization_lock(STM_SEGMENT->segment_num);
@@ -863,7 +865,8 @@
         realobj = REAL_ADDRESS(STM_SEGMENT->segment_base, obj);
         obj_size = stmcb_size_rounded_up((struct object_s *)realobj);
 
-        touch_all_pages_of_obj(obj, obj_size);
+        /* ACCESSIBLE, not READONLY since we need to write flags into the obj */
+        make_all_pages_of_obj_accessible(obj, obj_size);
     }
 
     if (mark_card) {
diff --git a/c8/stm/core.h b/c8/stm/core.h
--- a/c8/stm/core.h
+++ b/c8/stm/core.h
@@ -302,7 +302,7 @@
 static stm_thread_local_t *abort_with_mutex_no_longjmp(void);
 static void abort_data_structures_from_segment_num(int segment_num);
 
-static void touch_all_pages_of_obj(object_t *obj, size_t obj_size);
+static void make_all_pages_of_obj_accessible(object_t *obj, size_t obj_size);
 
 static void synchronize_object_enqueue(object_t *obj);
 static void synchronize_objects_flush(void);
diff --git a/c8/stm/gcpage.c b/c8/stm/gcpage.c
--- a/c8/stm/gcpage.c
+++ b/c8/stm/gcpage.c
@@ -249,6 +249,7 @@
             count++;
         }
     }
+    _assert_page_status_invariants(pagenum);
 
     return count;
 }
@@ -278,7 +279,7 @@
             fprintf(stderr, "COLLECT_HINTS_ONLY\n");
 
             /* XXX: since major GC also makes sure that currently modified objs are
-               "modified recently", maybe don't need to do it in touch_all_pages_of_obj() */
+               "modified recently", maybe don't need to do it in make_all_pages_of_obj_accessible() */
             return;
         }
     }
diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -671,7 +671,7 @@
     o->stm_flags = 0;
     /* make all pages of 'o' accessible as synchronize_obj_flush() in minor
        collections assumes all young objs are fully accessible. */
-    touch_all_pages_of_obj(o, size_rounded_up);
+    make_all_pages_of_obj_accessible(o, size_rounded_up);
 #endif
     return o;
 }
diff --git a/c8/stm/pages.h b/c8/stm/pages.h
--- a/c8/stm/pages.h
+++ b/c8/stm/pages.h
@@ -157,3 +157,41 @@
     ps->by_segment &= ~(0b11UL << seg_shift); /* clear */
     ps->by_segment |= status << seg_shift; /* set */
 }
+
+__attribute__((unused))
+static void _debug_page_status(uintptr_t pagenum)
+{
+    for (long i = 0; i < NB_SEGMENTS; i++) {
+        uint8_t status = get_page_status_in(i, pagenum);
+        switch (status) {
+        case PAGE_NO_ACCESS:
+            fprintf(stderr, "in seg %ld: NO_ACC\n", i);
+            break;
+        case PAGE_READONLY:
+            fprintf(stderr, "in seg %ld: RO\n", i);
+            break;
+        case PAGE_ACCESSIBLE:
+            fprintf(stderr, "in seg %ld: ACC\n", i);
+            break;
+        }
+    }
+}
+
+__attribute__((unused))
+static void _assert_page_status_invariants(uintptr_t pagenum)
+{
+#ifndef NDEBUG
+    bool has_ro = false;
+    bool has_acc = false;
+    for (long i = 1; i < NB_SEGMENTS; i++) {
+        uint8_t status = get_page_status_in(i, pagenum);
+        has_ro |= (status == PAGE_READONLY);
+        has_acc |= (status == PAGE_ACCESSIBLE);
+    }
+
+    if (has_ro && has_acc) {
+        _debug_page_status(pagenum);
+        assert(false);
+    }
+#endif
+}
diff --git a/c8/stm/signal_handler.c b/c8/stm/signal_handler.c
--- a/c8/stm/signal_handler.c
+++ b/c8/stm/signal_handler.c
@@ -5,7 +5,6 @@
 
 
 
-
 static void setup_signal_handler(void)
 {
     struct sigaction act;
@@ -61,7 +60,7 @@
 }
 
 
-static void readonly_to_accessible(int my_segnum, uintptr_t pagenum)
+static void any_to_accessible(int my_segnum, uintptr_t pagenum)
 {
     /* make our page write-ready */
     page_mark_accessible(my_segnum, pagenum);
@@ -91,22 +90,55 @@
 
     long i;
     int my_segnum = STM_SEGMENT->segment_num;
+    uint8_t page_status = get_page_status_in(my_segnum, pagenum);
 
-    uint8_t page_status = get_page_status_in(my_segnum, pagenum);
     assert(page_status == PAGE_NO_ACCESS
            || page_status == PAGE_READONLY);
+    _assert_page_status_invariants(pagenum);
 
-
-    //is_write=false;
-    if (page_status == PAGE_READONLY || is_write) {
-        dprintf(("SHORTCUT\n"));
-        readonly_to_accessible(my_segnum, pagenum);
-    }
     if (page_status == PAGE_READONLY) {
+        /* RO -> ACC */
+        assert(is_write);       /* should only fail if linux kernel changed */
+        any_to_accessible(my_segnum, pagenum);
         ro_to_acc++;
 
+        /* if was RO, page already has the right contents */
+        _assert_page_status_invariants(pagenum);
         release_all_privatization_locks();
         return;
+
+    }
+
+    assert(page_status == PAGE_NO_ACCESS);
+
+    /* if this is just a read-access, try to get a RO view: */
+    if (!is_write) {
+        bool acc_exists = false;
+        for (i = 1; i < NB_SEGMENTS; i++) {
+            if (i == my_segnum)
+                continue;
+
+            if (get_page_status_in(i, pagenum) == PAGE_ACCESSIBLE) {
+                acc_exists = true;
+                break;
+            }
+        }
+
+        if (!acc_exists) {
+            /* if there is no ACC version around, it means noone ever had that page
+             * ACC since the last major GC -> seg0 has the most current revision and
+             * we can get a RO of that. (of course only if this is not a
+             * write-access anyway) */
+
+            /* this case could be avoided by making all NO_ACCESS to READONLY
+               when resharing pages (XXX: better?).
+               We may go from NO_ACCESS->READONLY->ACCESSIBLE */
+            dprintf((" > make a previously NO_ACCESS page READONLY\n"));
+            page_mark_readonly(my_segnum, pagenum);
+            _assert_page_status_invariants(pagenum);
+            release_all_privatization_locks();
+            return;
+        }
     }
 
     /* find a suitable page to copy from in other segments:
@@ -116,26 +148,23 @@
      * Note: simply finding the most recent revision would be a conservative strategy, but
      *       requires going back in time more often (see below)
      */
+
+    /* special case: if there are RO versions around, we want to copy from seg0,
+     * since we make RO -> NOACC below before we copy (which wouldn't work). */
     int copy_from_segnum = -1;
     uint64_t copy_from_rev = 0;
     uint64_t target_rev = STM_PSEGMENT->last_commit_log_entry->rev_num;
-    bool was_readonly = false;
     for (i = 1; i < NB_SEGMENTS; i++) {
         if (i == my_segnum)
             continue;
 
-        if (!was_readonly && get_page_status_in(i, pagenum) == PAGE_READONLY) {
-            was_readonly = true;
-            break;
-        }
-
         struct stm_commit_log_entry_s *log_entry;
         log_entry = get_priv_segment(i)->last_commit_log_entry;
 
         /* - if not found anything, initialise copy_from_rev
          * - else if target_rev is higher than everything we found, find newest among them
          * - else: find revision that is as close to target_rev as possible         */
-        bool accessible = get_page_status_in(i, pagenum) != PAGE_NO_ACCESS;
+        bool accessible = get_page_status_in(i, pagenum) == PAGE_ACCESSIBLE;
         bool uninit = copy_from_segnum == -1;
         bool find_most_recent = copy_from_rev < target_rev && log_entry->rev_num > copy_from_rev;
         bool find_closest = copy_from_rev >= target_rev && (
@@ -150,29 +179,19 @@
     }
     OPT_ASSERT(copy_from_segnum != my_segnum);
 
-    if (was_readonly) {
-        assert(page_status == PAGE_NO_ACCESS);
-        /* this case could be avoided by making all NO_ACCESS to READONLY
-           when resharing pages (XXX: better?).
-           We may go from NO_ACCESS->READONLY->ACCESSIBLE on write with
-           2 SIGSEGV in a row.*/
-        dprintf((" > make a previously NO_ACCESS page READONLY\n"));
-        page_mark_readonly(my_segnum, pagenum);
 
-        release_all_privatization_locks();
-        return;
-    }
-
-    /* make our page write-ready */
-    if (!is_write) // is_write -> already marked accessible above
-        page_mark_accessible(my_segnum, pagenum);
+    /* make our page write-ready and reconstruct contents */
+    any_to_accessible(my_segnum, pagenum);
+    _assert_page_status_invariants(pagenum);
 
     /* account for this page now: XXX */
     /* increment_total_allocated(4096); */
 
     if (copy_from_segnum == -1) {
         /* this page is only accessible in the sharing segment seg0 so far (new
-           allocation). We can thus simply mark it accessible here. */
+           allocation). Or it was only in RO pages, which are the same as seg0.
+           We can thus simply mark it accessible here w/o undoing any
+           modifications or going back in time (seg0 is up-to-date). */
         pagecopy(get_virtual_page(my_segnum, pagenum),
                  get_virtual_page(0, pagenum));
         release_all_privatization_locks();
diff --git a/c8/test/test_resharing.py b/c8/test/test_resharing.py
--- a/c8/test/test_resharing.py
+++ b/c8/test/test_resharing.py
@@ -118,6 +118,62 @@
         assert stm_get_page_status(p2) == PAGE_NO_ACCESS
 
 
+    def test_seg0_updated_on_majorgc(self):
+        self.start_transaction()
+        lp1 = stm_allocate(16)
+        stm_set_char(lp1, 'a')
+        self.push_roots([lp1,])
+        stm_minor_collect()
+        lp1, = self.pop_roots()
+        self.push_roots([lp1,])
+        self.commit_transaction()
+        p1 = stm_get_obj_pages(lp1)[0]
+
+        self.switch(1)
+        self.start_transaction()
+        # NOACC -> ACC
+        assert stm_get_page_status(p1) == PAGE_NO_ACCESS
+        assert stm_get_char(lp1) == 'a'
+        assert stm_get_page_status(p1) == PAGE_ACCESSIBLE
+        stm_set_char(lp1, 'b')
+        self.commit_transaction()
+
+        self.switch(2)
+        self.start_transaction()
+        # NOACC -> ACC
+        assert stm_get_page_status(p1) == PAGE_NO_ACCESS
+        assert stm_get_char(lp1) == 'b'
+        # stm_set_char(lp1, 'x')
+        assert stm_get_page_status(p1) == PAGE_ACCESSIBLE
+        # merging: unmodified ACC -> RO
+        stm_major_collect()
+        stm_major_collect()
+        stm_major_collect()
+        assert stm_get_page_status(p1) == PAGE_READONLY
+
+        self.switch(1)
+
+        self.start_transaction()
+        # RO stays RO
+        assert stm_get_page_status(p1) == PAGE_READONLY
+        assert stm_get_char(lp1) == 'b'
+        assert stm_get_page_status(p1) == PAGE_READONLY
+
+        self.switch(2)
+
+        # write makes RO -> ACC, all others RO->NO_ACC
+        assert stm_get_page_status(p1) == PAGE_READONLY
+        stm_set_char(lp1, 'x')
+        assert stm_get_page_status(p1) == PAGE_ACCESSIBLE
+
+        self.switch(1)
+
+        assert stm_get_page_status(p1) == PAGE_NO_ACCESS
+        stm_major_collect()
+        stm_major_collect()
+        stm_major_collect()
+        assert stm_get_page_status(p1) == PAGE_NO_ACCESS
+
 
 
     def test_resharing_more(self):
@@ -215,7 +271,7 @@
         # now wref is in NO_ACCESS page
         assert stm_get_page_status(page) == PAGE_NO_ACCESS
         assert stm_get_weakref(wref) == ffi.NULL
-        assert stm_get_page_status(page) == PAGE_ACCESSIBLE
+        assert stm_get_page_status(page) == PAGE_READONLY
 
     def test_weakref2(self):
         self.start_transaction()
@@ -244,4 +300,4 @@
         # now wref is in NO_ACCESS page
         assert stm_get_page_status(page) == PAGE_NO_ACCESS
         assert stm_get_weakref(wref) == ffi.NULL
-        assert stm_get_page_status(page) == PAGE_ACCESSIBLE
+        assert stm_get_page_status(page) == PAGE_READONLY

From pypy.commits at gmail.com  Wed Jun 14 14:41:24 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 14 Jun 2017 11:41:24 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Implement fill_slot()
Message-ID: <59418354.85251c0a.e385a.456a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91605:638043b85bc5
Date: 2017-06-14 19:40 +0100
http://bitbucket.org/pypy/pypy/changeset/638043b85bc5/

Log:	Implement fill_slot()

diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -154,7 +154,7 @@
         ex = module.Example()
         assert ex.demo('abcd') == 'abcd'
         assert ex.demo() is None
-        raises(AttributeError, ex.abc)
+        raises(AttributeError, 'ex.abc')
         ex.abc = 0
         assert ex.abc == 0
         assert module.foo(9, 9) == 18
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -1,5 +1,6 @@
 import os
 
+from rpython.rlib.unroll import unrolling_iterable
 from rpython.rlib import jit
 from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rtyper.lltypesystem import rffi, lltype
@@ -17,7 +18,7 @@
     build_type_checkers, Py_TPFLAGS_BASETYPE,
     PyObjectFields, PyTypeObject, PyTypeObjectPtr,
     cts, parse_dir)
-from pypy.module.cpyext.cparser import parse_source
+from pypy.module.cpyext.cparser import CTypeSpace
 from pypy.module.cpyext.methodobject import (W_PyCClassMethodObject,
     W_PyCWrapperObject, PyCFunction_NewEx, PyCFunction, PyMethodDef,
     W_PyCMethodObject, W_PyCFunctionObject)
@@ -41,6 +42,7 @@
 PyType_Check, PyType_CheckExact = build_type_checkers("Type", "w_type")
 
 PyHeapTypeObject = cts.gettype('PyHeapTypeObject *')
+
 cts.parse_header(parse_dir / "typeslots.h")
 
 
@@ -867,6 +869,36 @@
     return generic_cpy_call(
         space, type.c_tp_alloc, type, 0)
 
+def _parse_typeslots():
+    slots_hdr = CTypeSpace()
+    slots_hdr.parse_header(parse_dir / "typeslots.h")
+    prefix2member = {
+        'tp': "ht_type",
+        'am': "as_async",
+        'nb': "as_number",
+        'mp': "as_mapping",
+        'sq': "as_sequence",
+        'bf': "as_buffer"}
+
+    TABLE = []
+    HTO = cts.gettype('PyHeapTypeObject')
+    for name, num in slots_hdr.macros.items():
+        assert isinstance(num, int)
+        assert name.startswith('Py_')
+        name = name[3:]
+        membername = 'c_' + prefix2member[name[:2]]
+        slotname = 'c_' + name
+        TARGET = HTO._flds[membername]._flds[slotname]
+        TABLE.append((num, membername, slotname, TARGET))
+    return unrolling_iterable(TABLE)
+SLOT_TABLE = _parse_typeslots()
+
+def fill_slot(ht, slotnum, ptr):
+    for num, membername, slotname, TARGET in SLOT_TABLE:
+        if num == slotnum:
+            setattr(getattr(ht, membername), slotname, rffi.cast(TARGET, ptr))
+
+
 @cts.decl("""PyObject *
     PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)""",
     result_is_ll=True)
@@ -879,7 +911,7 @@
     #s = 'foo'
     #res.c_ht_name = PyUnicode_FromString(s)
     #res.c_ht_qualname = res.c_ht_name
-    incref(space, res.c_ht_qualname)
+    #incref(space, res.c_ht_qualname)
     typ.c_tp_name = spec.c_name
     slotdefs = rffi.cast(rffi.CArrayPtr(cts.gettype('PyType_Slot')), spec.c_slots)
     if not bases:
@@ -899,7 +931,7 @@
         if not bases_w:
             bases_w = [w_base]
     else:
-        bases_w = space.fixed_view(from_ref(space, bases))
+        bases_w = space.fixedview(from_ref(space, bases))
     w_base = best_base(space, bases_w)
     base = cts.cast('PyTypeObject*', make_ref(space, w_base))
     if False:  # not base.c_tp_flags & Py_TPFLAGS_BASETYPE:
@@ -929,7 +961,7 @@
             # Processed above
             i += 1
             continue
-        #fill_slot(res, slot, slotdef.c_pfunc)
+        fill_slot(res, slot, slotdef.c_pfunc)
         # XXX: need to make a copy of the docstring slot, which usually
         # points to a static string literal
         i += 1

From pypy.commits at gmail.com  Wed Jun 14 15:26:00 2017
From: pypy.commits at gmail.com (fijal)
Date: Wed, 14 Jun 2017 12:26:00 -0700 (PDT)
Subject: [pypy-commit] extradoc extradoc: abstract
Message-ID: <59418dc8.cb7e1c0a.3fef0.5a24@mx.google.com>

Author: fijal
Branch: extradoc
Changeset: r5805:c7c6ff9479eb
Date: 2017-06-14 21:25 +0200
http://bitbucket.org/pypy/extradoc/changeset/c7c6ff9479eb/

Log:	abstract

diff --git a/talk/pycon-russia-2017/abstract.rst b/talk/pycon-russia-2017/abstract.rst
new file mode 100644
--- /dev/null
+++ b/talk/pycon-russia-2017/abstract.rst
@@ -0,0 +1,15 @@
+Why is Python slow?
+===================
+
+In this talk, I will cover one of the usual topics of discussions
+of deficiencies of the Python language, notably it's speed. We've spend
+last decade improving on the performance of the language (but not the default
+implementation). What we've learned so far is not what's the common
+interpretation of that question - things like dynamic dispatch, dynamic
+typing and interpreter can all be worked around. What we learned are the
+places in the Python language which make unnecessary string copies, but
+are easier to write, a lot of "pythonic" constructs that cannot be implemented
+efficiently and a lot of quirks that make some constructs slow for no good
+reason. Additionally we're overwhelmed by the sheer size of the "simple"
+language and the necessity of supporting C extensions. This talk will cover
+detailed view of those problems and some potential remedies.
\ No newline at end of file

From pypy.commits at gmail.com  Thu Jun 15 04:20:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Jun 2017 01:20:38 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #319
Message-ID: <59424356.dc361c0a.23123.fd50@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2977:75275c539e51
Date: 2017-06-15 10:19 +0200
http://bitbucket.org/cffi/cffi/changeset/75275c539e51/

Log:	Issue #319

	Fix a race condition: we can't stick an attribute to ffi._backend,
	because another thread might be in the middle of ``for name in
	ffi._backend.__dict__``.

	To minimize damage if I reintroduce an attribute-adding function in
	the future, also tweak the ``for``.

diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -75,9 +75,10 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
+        self._typecache = model.get_typecache(backend)
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
-        for name in backend.__dict__:
+        for name in list(backend.__dict__):
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
diff --git a/cffi/model.py b/cffi/model.py
--- a/cffi/model.py
+++ b/cffi/model.py
@@ -568,22 +568,26 @@
 
 
 global_lock = allocate_lock()
+_typecache_cffi_backend = weakref.WeakValueDictionary()
+
+def get_typecache(backend):
+    # returns _typecache_cffi_backend if backend is the _cffi_backend
+    # module, or type(backend).__typecache if backend is an instance of
+    # CTypesBackend (or some FakeBackend class during tests)
+    if isinstance(backend, types.ModuleType):
+        return _typecache_cffi_backend
+    with global_lock:
+        if not hasattr(type(backend), '__typecache'):
+            type(backend).__typecache = weakref.WeakValueDictionary()
+        return type(backend).__typecache
 
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
     try:
-        return ffi._backend.__typecache[key]
+        return ffi._typecache[key]
     except KeyError:
         pass
-    except AttributeError:
-        # initialize the __typecache attribute, either at the module level
-        # if ffi._backend is a module, or at the class level if ffi._backend
-        # is some instance.
-        if isinstance(ffi._backend, types.ModuleType):
-            ffi._backend.__typecache = weakref.WeakValueDictionary()
-        else:
-            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
@@ -591,7 +595,7 @@
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
-    cache = ffi._backend.__typecache
+    cache = ffi._typecache
     with global_lock:
         res1 = cache.get(key)
         if res1 is None:

From pypy.commits at gmail.com  Thu Jun 15 05:26:17 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Jun 2017 02:26:17 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #1853
Message-ID: <594252b9.c4581c0a.492b8.0644@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91606:0b207c82e5fd
Date: 2017-06-15 11:25 +0200
http://bitbucket.org/pypy/pypy/changeset/0b207c82e5fd/

Log:	Issue #1853

	Add cPickle.Unpickler.find_global

diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -431,7 +431,14 @@
         self.append(obj)
 
     def find_class(self, module, name):
-        # Subclasses may override this
+        if self.find_global is None:
+            raise UnpicklingError(
+                "Global and instance pickles are not supported.")
+        return self.find_global(module, name)
+
+    def find_global(self, module, name):
+        # This can officially be patched directly in the Unpickler
+        # instance, according to the docs
         __import__(module)
         mod = sys.modules[module]
         klass = getattr(mod, name)
diff --git a/pypy/module/test_lib_pypy/test_cPickle.py b/pypy/module/test_lib_pypy/test_cPickle.py
--- a/pypy/module/test_lib_pypy/test_cPickle.py
+++ b/pypy/module/test_lib_pypy/test_cPickle.py
@@ -9,3 +9,25 @@
 def test_bad_key():
     e = py.test.raises(cPickle.UnpicklingError, cPickle.loads, "v")
     assert str(e.value) == "invalid load key, 'v'."
+
+def test_find_global():
+    import time, cStringIO
+    entry = time.strptime('Fri Mar 27 22:20:42 2017')
+    f = cStringIO.StringIO()
+    cPickle.Pickler(f).dump(entry)
+
+    f = cStringIO.StringIO(f.getvalue())
+    e = cPickle.Unpickler(f).load()
+    assert e == entry
+
+    f = cStringIO.StringIO(f.getvalue())
+    up = cPickle.Unpickler(f)
+    up.find_global = None
+    e = py.test.raises(cPickle.UnpicklingError, up.load)
+    assert str(e.value) == "Global and instance pickles are not supported."
+
+    f = cStringIO.StringIO(f.getvalue())
+    up = cPickle.Unpickler(f)
+    up.find_global = lambda module, name: lambda a, b: (a, b)
+    e = up.load()
+    assert e == ((2017, 3, 27, 22, 20, 42, 4, 86, -1), {})

From pypy.commits at gmail.com  Thu Jun 15 05:29:04 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 15 Jun 2017 02:29:04 -0700 (PDT)
Subject: [pypy-commit] pypy default: improve test
Message-ID: <59425360.c2a0df0a.30bb0.8232@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91607:4a32e5d6f559
Date: 2017-06-15 11:28 +0200
http://bitbucket.org/pypy/pypy/changeset/4a32e5d6f559/

Log:	improve test

diff --git a/pypy/module/test_lib_pypy/test_cPickle.py b/pypy/module/test_lib_pypy/test_cPickle.py
--- a/pypy/module/test_lib_pypy/test_cPickle.py
+++ b/pypy/module/test_lib_pypy/test_cPickle.py
@@ -28,6 +28,6 @@
 
     f = cStringIO.StringIO(f.getvalue())
     up = cPickle.Unpickler(f)
-    up.find_global = lambda module, name: lambda a, b: (a, b)
+    up.find_global = lambda module, name: lambda a, b: (name, a, b)
     e = up.load()
-    assert e == ((2017, 3, 27, 22, 20, 42, 4, 86, -1), {})
+    assert e == ('struct_time', (2017, 3, 27, 22, 20, 42, 4, 86, -1), {})

From pypy.commits at gmail.com  Thu Jun 15 10:28:17 2017
From: pypy.commits at gmail.com (tobweber)
Date: Thu, 15 Jun 2017 07:28:17 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-binary-trx-length-per-thread: Set short
 transaction length to default
Message-ID: <59429981.87a4df0a.c4f4e.264a@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-binary-trx-length-per-thread
Changeset: r2076:472cd93faa7e
Date: 2017-06-13 15:10 +0200
http://bitbucket.org/pypy/stmgc/changeset/472cd93faa7e/

Log:	Set short transaction length to default

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -22,7 +22,7 @@
     double new = tl->relative_transaction_length;
     if (aborts) {
         tl->transaction_length_backoff = 3;
-        new = 100.0 / LARGE_FILL_MARK_NURSERY_BYTES;
+        new = (double)DEFAULT_FILL_MARK_NURSERY_BYTES / LARGE_FILL_MARK_NURSERY_BYTES;
     } else if (tl->transaction_length_backoff == 0) {
         new = 1;
     } else { // not abort and backoff != 0

From pypy.commits at gmail.com  Thu Jun 15 10:28:14 2017
From: pypy.commits at gmail.com (tobweber)
Date: Thu, 15 Jun 2017 07:28:14 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-binary-trx-length-per-thread: Implement
 simple binary adaptive transaction length
Message-ID: <5942997e.0f1f1c0a.7fbe3.25de@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-binary-trx-length-per-thread
Changeset: r2074:abbe431338a8
Date: 2017-06-11 18:26 +0200
http://bitbucket.org/pypy/stmgc/changeset/abbe431338a8/

Log:	Implement simple binary adaptive transaction length

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -16,30 +16,15 @@
 static uintptr_t _stm_nursery_start;
 
 #define DEFAULT_FILL_MARK_NURSERY_BYTES (NURSERY_SIZE / 4)
-
-// #define LARGE_FILL_MARK_NURSERY_BYTES   DEFAULT_FILL_MARK_NURSERY_BYTES
 #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000L
-// #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
-
-#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001)
 
 static double get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
-    const int multiplier = 100;
-    double previous = tl->relative_transaction_length;
-    double new = previous;
+    double new = tl->relative_transaction_length;
     if (aborts) {
         tl->transaction_length_backoff = 3;
-        if (previous > STM_MIN_RELATIVE_TRANSACTION_LENGTH) {
-            new = previous / multiplier;
-        } else {
-            new = 0;
-        }
+        new = 0;
     } else if (tl->transaction_length_backoff == 0) {
-        if (previous - (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1) < 0) {
-            new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
-        } else if (previous < 1) {
-            new = previous * multiplier;
-        }
+        new = 1;
     } else { // not abort and backoff != 0
         tl->transaction_length_backoff -= 1;
     }
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -247,7 +247,7 @@
     tl->thread_local_counter = ++thread_local_counters;
 
     /* init single thread mode */
-    tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
+    tl->relative_transaction_length = 1;
     tl->transaction_length_backoff = 0;
 
     *_get_cpth(tl) = pthread_self();

From pypy.commits at gmail.com  Thu Jun 15 10:28:16 2017
From: pypy.commits at gmail.com (tobweber)
Date: Thu, 15 Jun 2017 07:28:16 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-binary-trx-length-per-thread: Set short
 transaction length greater zero
Message-ID: <59429980.e492df0a.9c2fa.2072@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-binary-trx-length-per-thread
Changeset: r2075:953006e6836b
Date: 2017-06-12 09:25 +0200
http://bitbucket.org/pypy/stmgc/changeset/953006e6836b/

Log:	Set short transaction length greater zero

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -22,7 +22,7 @@
     double new = tl->relative_transaction_length;
     if (aborts) {
         tl->transaction_length_backoff = 3;
-        new = 0;
+        new = 100.0 / LARGE_FILL_MARK_NURSERY_BYTES;
     } else if (tl->transaction_length_backoff == 0) {
         new = 1;
     } else { // not abort and backoff != 0

From pypy.commits at gmail.com  Thu Jun 15 10:28:19 2017
From: pypy.commits at gmail.com (tobweber)
Date: Thu, 15 Jun 2017 07:28:19 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-binary-trx-length-per-thread: Select short
 trx length to be 100B
Message-ID: <59429983.c385df0a.b8200.25c4@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-binary-trx-length-per-thread
Changeset: r2077:e264072fb063
Date: 2017-06-15 16:27 +0200
http://bitbucket.org/pypy/stmgc/changeset/e264072fb063/

Log:	Select short trx length to be 100B

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -22,7 +22,7 @@
     double new = tl->relative_transaction_length;
     if (aborts) {
         tl->transaction_length_backoff = 3;
-        new = (double)DEFAULT_FILL_MARK_NURSERY_BYTES / LARGE_FILL_MARK_NURSERY_BYTES;
+        new = 100.0 / LARGE_FILL_MARK_NURSERY_BYTES;
     } else if (tl->transaction_length_backoff == 0) {
         new = 1;
     } else { // not abort and backoff != 0

From pypy.commits at gmail.com  Thu Jun 15 11:03:34 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 15 Jun 2017 08:03:34 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: Add more tests;
 actually set the type name in PyType_FromSpec()
Message-ID: <5942a1c6.87a4df0a.c4f4e.3271@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91608:eb06701ddfd6
Date: 2017-06-15 16:02 +0100
http://bitbucket.org/pypy/pypy/changeset/eb06701ddfd6/

Log:	Add more tests; actually set the type name in PyType_FromSpec()

diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -158,4 +158,16 @@
         ex.abc = 0
         assert ex.abc == 0
         assert module.foo(9, 9) == 18
+        assert isinstance(module.Str(), str)
+        assert module.Str(1) + '23' == '123'
+        raises(module.error, 'raise module.error()')
+        assert module.int_const == 1969
+        assert module.str_const == 'something different'
 
+    def test_reload(self):
+        import importlib
+        NAME = '_testmultiphase'
+        module = self.import_module(name=NAME)
+        ex_class = module.Example
+        importlib.reload(module)
+        assert ex_class is module.Example
diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -903,15 +903,22 @@
     PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases)""",
     result_is_ll=True)
 def PyType_FromSpecWithBases(space, spec, bases):
+    from pypy.module.cpyext.unicodeobject import PyUnicode_FromString
     res = PyType_GenericAlloc(space, space.w_type, 0)
     res = cts.cast('PyHeapTypeObject *', res)
     typ = res.c_ht_type
     typ.c_tp_flags = rffi.cast(lltype.Signed, spec.c_flags)
     typ.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
-    #s = 'foo'
-    #res.c_ht_name = PyUnicode_FromString(s)
-    #res.c_ht_qualname = res.c_ht_name
-    #incref(space, res.c_ht_qualname)
+    specname = rffi.charp2str(spec.c_name)
+    dotpos = specname.rfind('.')
+    if dotpos < 0:
+        name = specname
+    else:
+        name = specname[dotpos + 1:]
+    res.c_ht_name = make_ref(
+        space, PyUnicode_FromString(space, rffi.str2charp(name)))
+    res.c_ht_qualname = res.c_ht_name
+    incref(space, res.c_ht_qualname)
     typ.c_tp_name = spec.c_name
     slotdefs = rffi.cast(rffi.CArrayPtr(cts.gettype('PyType_Slot')), spec.c_slots)
     if not bases:

From pypy.commits at gmail.com  Thu Jun 15 12:10:31 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 15 Jun 2017 09:10:31 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: fix translation
Message-ID: <5942b177.894a1c0a.c9f42.416d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91609:bb2c4b519f09
Date: 2017-06-15 17:09 +0100
http://bitbucket.org/pypy/pypy/changeset/bb2c4b519f09/

Log:	fix translation

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -909,7 +909,7 @@
     typ = res.c_ht_type
     typ.c_tp_flags = rffi.cast(lltype.Signed, spec.c_flags)
     typ.c_tp_flags |= Py_TPFLAGS_HEAPTYPE
-    specname = rffi.charp2str(spec.c_name)
+    specname = rffi.charp2str(cts.cast('char*', spec.c_name))
     dotpos = specname.rfind('.')
     if dotpos < 0:
         name = specname

From pypy.commits at gmail.com  Thu Jun 15 14:19:50 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 15 Jun 2017 11:19:50 -0700 (PDT)
Subject: [pypy-commit] pypy default: ignore more local status files
Message-ID: <5942cfc6.58abdf0a.01e1.647c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91610:52f2b6e7ff47
Date: 2017-06-15 20:26 +0300
http://bitbucket.org/pypy/pypy/changeset/52f2b6e7ff47/

Log:	ignore more local status files

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -8,6 +8,8 @@
 .pydevproject
 __pycache__
 
+.cache/
+.gdb_history
 syntax: regexp
 ^testresult$
 ^site-packages$
@@ -85,7 +87,6 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
-^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
 

From pypy.commits at gmail.com  Thu Jun 15 14:19:52 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 15 Jun 2017 11:19:52 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branches
Message-ID: <5942cfc8.ce0c1c0a.e73f4.5ebb@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91611:b491b26971bf
Date: 2017-06-15 21:06 +0300
http://bitbucket.org/pypy/pypy/changeset/b491b26971bf/

Log:	document merged branches

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,11 @@
 .. branch: cffi-char16-char32
 
 The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
+
+.. branch: ctypes_char_indexing
+
+Indexing into char* behaves differently than CPython
+
+.. branch: vmprof-0.4.8
+
+Improve and fix issues with vmprof

From pypy.commits at gmail.com  Thu Jun 15 14:33:24 2017
From: pypy.commits at gmail.com (ntruessel)
Date: Thu, 15 Jun 2017 11:33:24 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Fix -O0 not working with qcgc
Message-ID: <5942d2f4.d18bdf0a.bbeea.640c@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r91612:e1122de6d5cc
Date: 2017-06-15 20:31 +0200
http://bitbucket.org/pypy/pypy/changeset/e1122de6d5cc/

Log:	Fix -O0 not working with qcgc

diff --git a/rpython/config/translationoption.py b/rpython/config/translationoption.py
--- a/rpython/config/translationoption.py
+++ b/rpython/config/translationoption.py
@@ -77,9 +77,9 @@
                                ("translation.gctransformer", "boehm")],
                      "qcgc": [("translation.gctransformer", "framework"),
                               ("translation.gcrootfinder", "qcgc"),
-                              ("translation.gcremovetypeptr", True),
+                              ("translation.gcremovetypeptr", False),
                               ("translation.thread", False),
-                              ("translation.rweakref", True)], # XXX
+                              ("translation.rweakref", True)],
                      "minimark": [("translation.gctransformer", "framework")],
                      "incminimark": [("translation.gctransformer", "framework")],
                      },

From pypy.commits at gmail.com  Fri Jun 16 02:44:08 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 15 Jun 2017 23:44:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: support PyFrozenSet_New
Message-ID: <59437e38.53061c0a.b7b6d.8a6f@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91613:776f717d31e0
Date: 2017-06-16 09:43 +0300
http://bitbucket.org/pypy/pypy/changeset/776f717d31e0/

Log:	support PyFrozenSet_New

diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -9,6 +9,7 @@
 
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
+PyFrozenSet_Check, PyFrozenSet_CheckExact = build_type_checkers("FrozenSet")
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyAnySet_Check(space, w_obj):
@@ -91,7 +92,7 @@
     """Return the length of a set or frozenset object. Equivalent to
     len(anyset).  Raises a PyExc_SystemError if anyset is not a set, frozenset,
     or an instance of a subtype."""
-    if not PySet_Check(space, ref):
+    if not PyAnySet_Check(space, ref):
         raise oefmt(space.w_TypeError, "expected set object")
     return PySet_GET_SIZE(space, ref)
 
@@ -104,3 +105,20 @@
     set, frozenset, or an instance of a subtype."""
     w_res = space.contains(w_obj, w_key)
     return space.int_w(w_res)
+
+ at cpython_api([PyObject], PyObject)
+def PyFrozenSet_New(space, w_iterable):
+    """Return a new frozenset containing objects returned by the iterable.
+    The iterable may be NULL to create a new empty frozenset.  Return the new
+    set on success or NULL on failure.  Raise TypeError if iterable is
+    not actually iterable.
+
+    Now guaranteed to return a brand-new frozenset.  Formerly,
+    frozensets of zero-length were a singleton.  This got in the way of
+    building-up new frozensets with PySet_Add()."""
+    if w_iterable is None:
+        return space.call_function(space.w_frozenset)
+    else:
+        return space.call_function(space.w_frozenset, w_iterable)
+
+
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1423,31 +1423,6 @@
     in your code for properly supporting 64-bit systems."""
     raise NotImplementedError
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyFrozenSet_Check(space, p):
-    """Return true if p is a frozenset object or an instance of a
-    subtype.
-    """
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyFrozenSet_CheckExact(space, p):
-    """Return true if p is a frozenset object but not an instance of a
-    subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], PyObject)
-def PyFrozenSet_New(space, iterable):
-    """Return a new frozenset containing objects returned by the iterable.
-    The iterable may be NULL to create a new empty frozenset.  Return the new
-    set on success or NULL on failure.  Raise TypeError if iterable is
-    not actually iterable.
-
-    Now guaranteed to return a brand-new frozenset.  Formerly,
-    frozensets of zero-length were a singleton.  This got in the way of
-    building-up new frozensets with PySet_Add()."""
-    raise NotImplementedError
-
 @cpython_api([rffi.CCHARP, Py_ssize_t, rffi.CCHARP, rffi.CCHARP], PyObject)
 def PyString_Decode(space, s, size, encoding, errors):
     """Create an object by decoding size bytes of the encoded buffer s using the
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -9,9 +9,11 @@
 class TestTupleObject(BaseApiTest):
     def test_setobj(self, space, api):
         assert not api.PySet_Check(space.w_None)
+        assert not api.PyFrozenSet_Check(space.w_None)
         assert api.PySet_Add(space.w_None, space.w_None) == -1
         api.PyErr_Clear()
         w_set = space.call_function(space.w_set)
+        assert not api.PyFrozenSet_CheckExact(w_set)
         space.call_method(w_set, 'update', space.wrap([1,2,3,4]))
         assert api.PySet_Size(w_set) == 4
         assert api.PySet_GET_SIZE(w_set) == 4
@@ -21,6 +23,8 @@
     def test_set_add_discard(self, space, api):
         w_set = api.PySet_New(None)
         assert api.PySet_Size(w_set) == 0
+        w_set = api.PyFrozenSet_New(space.wrap([1,2,3,4]))
+        assert api.PySet_Size(w_set) == 4
         w_set = api.PySet_New(space.wrap([1,2,3,4]))
         assert api.PySet_Size(w_set) == 4
         api.PySet_Add(w_set, space.wrap(6))

From pypy.commits at gmail.com  Fri Jun 16 09:12:38 2017
From: pypy.commits at gmail.com (ntruessel)
Date: Fri, 16 Jun 2017 06:12:38 -0700 (PDT)
Subject: [pypy-commit] pypy quad-color-gc: Update QCGC
Message-ID: <5943d946.5aa0df0a.d04de.fc8a@mx.google.com>

Author: Nicolas Truessel <ntruessel at njsm.de>
Branch: quad-color-gc
Changeset: r91614:ef5bd5ed6279
Date: 2017-06-16 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/ef5bd5ed6279/

Log:	Update QCGC

diff --git a/rpython/translator/c/src/qcgc/qcgc.c b/rpython/translator/c/src/qcgc/qcgc.c
--- a/rpython/translator/c/src/qcgc/qcgc.c
+++ b/rpython/translator/c/src/qcgc/qcgc.c
@@ -88,8 +88,8 @@
 		}
 	}
 
-	// FIXME: alligned_alloc requires size to be a multiple of the alignment
-	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, size);
+	size_t rounded_size = (size + QCGC_ARENA_SIZE - 1) & ~(QCGC_ARENA_SIZE - 1);
+	object_t *result = aligned_alloc(QCGC_ARENA_SIZE, rounded_size);
 #if QCGC_INIT_ZERO
 	memset(result, 0, size);
 #endif

From pypy.commits at gmail.com  Fri Jun 16 10:55:43 2017
From: pypy.commits at gmail.com (stefanor)
Date: Fri, 16 Jun 2017 07:55:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Some more (obscure) architectures that
 would like -fPIC
Message-ID: <5943f16f.d9421c0a.e6c37.3b91@mx.google.com>

Author: Stefano Rivera <stefano at rivera.za.net>
Branch: 
Changeset: r91615:ad6889740dc0
Date: 2017-06-16 07:55 -0700
http://bitbucket.org/pypy/pypy/changeset/ad6889740dc0/

Log:	Some more (obscure) architectures that would like -fPIC

diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -260,7 +260,8 @@
     # Only required on armhf and mips{,el}, not armel. But there's no way to
     # detect armhf without shelling out
     if (platform.architecture()[0] == '64bit'
-            or platform.machine().startswith(('arm', 'mips', 'ppc'))):
+            or platform.machine().startswith(
+                ('arm', 'm68k', 'mips', 'parisc', 'ppc', 'sh4'))):
         host_factory = LinuxPIC
     else:
         host_factory = Linux

From pypy.commits at gmail.com  Fri Jun 16 11:28:33 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Jun 2017 08:28:33 -0700 (PDT)
Subject: [pypy-commit] cffi default: Document issue #319
Message-ID: <5943f921.e793df0a.5f402.a460@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2978:78e1cba114e1
Date: 2017-06-16 12:04 +0200
http://bitbucket.org/cffi/cffi/changeset/78e1cba114e1/

Log:	Document issue #319

diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -20,6 +20,12 @@
   mode you still cannot call C functions that take complex numbers
   directly as arguments or return type.
 
+* Fixed a rare race condition when creating multiple ``FFI`` instances
+  from multiple threads.  (Note that you aren't meant to create many
+  ``FFI`` instances: in inline mode, you should write ``ffi =
+  cffi.FFI()`` at module level just after ``import cffi``; and in
+  out-of-line mode you don't instantiate ``FFI`` explicitly at all.)
+
 
 v1.10.1
 =======

From pypy.commits at gmail.com  Fri Jun 16 11:28:35 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Jun 2017 08:28:35 -0700 (PDT)
Subject: [pypy-commit] cffi default: Basic "does not crash" test for the
 error logic in _cffi_initialize_python
Message-ID: <5943f923.2196df0a.69b19.b1ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2979:b0f094e3f3eb
Date: 2017-06-16 13:44 +0200
http://bitbucket.org/cffi/cffi/changeset/b0f094e3f3eb/

Log:	Basic "does not crash" test for the error logic in
	_cffi_initialize_python

diff --git a/testing/embedding/initerror.py b/testing/embedding/initerror.py
new file mode 100644
--- /dev/null
+++ b/testing/embedding/initerror.py
@@ -0,0 +1,18 @@
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+    int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+    raise KeyError
+""")
+
+ffi.set_source("_initerror_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
+
diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py
--- a/testing/embedding/test_basic.py
+++ b/testing/embedding/test_basic.py
@@ -199,3 +199,9 @@
                           "prepADD2\n"
                           "adding 100 and -5 and -20\n"
                           "got: 42 75\n")
+
+    def test_init_time_error(self):
+        initerror_cffi = self.prepare_module('initerror')
+        self.compile('add1-test', [initerror_cffi])
+        output = self.execute('add1-test')
+        assert output == "got: 0 0\n"    # plus lots of info to stderr

From pypy.commits at gmail.com  Fri Jun 16 11:28:37 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Jun 2017 08:28:37 -0700 (PDT)
Subject: [pypy-commit] cffi default: Windows: when a CFFI embedding attempt
 fails, attempt to
Message-ID: <5943f925.88b51c0a.3959b.9ae3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2980:cff66142f09b
Date: 2017-06-16 17:28 +0200
http://bitbucket.org/cffi/cffi/changeset/cff66142f09b/

Log:	Windows: when a CFFI embedding attempt fails, attempt to open a non-
	modal MessageBox() in addition to sending the detailed error to
	stderr... because stderr is likely to go nowhere, in case of non-
	console applications.

diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -110,6 +110,133 @@
 #ifndef PYPY_VERSION
 
 
+#ifdef _MSC_VER
+/* Windows only: logic to take the Python-CFFI embedding logic
+   initialization errors and display them in a background thread
+   with MessageBox.  The idea is that if the whole program closes
+   as a result of this problem, then likely it is already a console
+   program and you can read the stderr output in the console too.
+   If it is not a console program, then it will likely show its own
+   dialog to complain, or generally not abruptly close, and for this
+   case the background thread should stay alive.
+*/
+static PyObject *_cffi_bootstrap_string;
+static void *volatile _cffi_bootstrap_text;
+
+static PyObject *_cffi_start_error_capture(void)
+{
+    PyObject *result = NULL;
+    PyObject *x, *m, *bi;
+
+    if (!cffi_compare_and_swap(&_cffi_bootstrap_text, NULL, (void *)1))
+        return (PyObject *)1;
+
+    m = PyImport_AddModule("_cffi_error_capture");
+    if (m == NULL)
+        goto error;
+
+    result = PyModule_GetDict(m);
+    if (result == NULL)
+        goto error;
+
+    bi = PyImport_ImportModule("__builtin__");
+    if (bi == NULL)
+        goto error;
+    PyDict_SetItemString(result, "__builtins__", bi);
+    Py_DECREF(bi);
+
+    x = PyRun_String(
+        "import sys\n"
+        "class FileLike:\n"
+        "  def write(self, x):\n"
+        "    sys.__stderr__.write(x)\n"
+        "    self.buf += x\n"
+        "fl = FileLike()\n"
+        "fl.buf = ''\n"
+        "sys.stderr = fl\n"
+        "def done():\n"
+        "  sys.stderr = sys.__stderr__\n"
+        "  return fl.buf\n",   /* make sure the returned value stays alive */
+        Py_file_input,
+        result, result);
+    Py_XDECREF(x);
+
+ error:
+    if (PyErr_Occurred())
+    {
+        PyErr_WriteUnraisable(Py_None);
+        PyErr_Clear();
+    }
+    return result;
+}
+
+#pragma comment(lib, "user32.lib")
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+#if PY_MAJOR_VERSION >= 3
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"Python-CFFI embedding error",
+                MB_OK | MB_ICONERROR);
+#else
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "Python-CFFI embedding error",
+                MB_OK | MB_ICONERROR);
+#endif
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+static void _cffi_stop_error_capture(PyObject *ecap)
+{
+    PyObject *s;
+    void *text;
+
+    if (ecap == (PyObject *)1)
+        return;
+
+    if (ecap == NULL)
+        goto error;
+
+    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
+    if (s == NULL)
+        goto error;
+
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+#if PY_MAJOR_VERSION >= 3
+    text = PyUnicode_AsWideCharString(s, NULL);
+#else
+    text = PyString_AsString(s);
+#endif
+
+    _cffi_bootstrap_text = text;
+
+    if (text != NULL)
+    {
+        HANDLE h;
+        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                         NULL, 0, NULL);
+        if (h != NULL)
+            CloseHandle(h);
+    }
+    /* decref the string, but it should stay alive as 'fl.buf'
+       in the small module above.  It will really be freed only if
+       we later get another similar error.  So it's a leak of at
+       most one copy of the small module.  That's fine for this
+       situation which is usually a "fatal error" anyway. */
+    Py_DECREF(s);
+    PyErr_Clear();
+    return;
+
+  error:
+    _cffi_bootstrap_text = NULL;
+    PyErr_Clear();
+}
+#endif
+
+
 #define _cffi_call_python_org  _cffi_exports[_CFFI_CPIDX]
 
 PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void);   /* forward */
@@ -220,8 +347,20 @@
         /* Print as much information as potentially useful.
            Debugging load-time failures with embedding is not fun
         */
+#ifdef _MSC_VER
+        PyObject *ecap;
+#endif
         PyObject *exception, *v, *tb, *f, *modules, *mod;
         PyErr_Fetch(&exception, &v, &tb);
+#ifdef _MSC_VER
+        ecap = _cffi_start_error_capture();
+#endif
+        f = PySys_GetObject((char *)"stderr");
+        if (f != NULL && f != Py_None) {
+            PyFile_WriteString(
+                "Failed to initialize the Python-CFFI embedding logic:\n\n", f);
+        }
+
         if (exception != NULL) {
             PyErr_NormalizeException(&exception, &v, &tb);
             PyErr_Display(exception, v, tb);
@@ -230,7 +369,6 @@
         Py_XDECREF(v);
         Py_XDECREF(tb);
 
-        f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
                                "\ncompiled with cffi version: 1.11.0"
@@ -249,6 +387,9 @@
             PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
             PyFile_WriteString("\n\n", f);
         }
+#ifdef _MSC_VER
+        _cffi_stop_error_capture(ecap);
+#endif
     }
     result = -1;
     goto done;
diff --git a/testing/embedding/add1-test.c b/testing/embedding/add1-test.c
--- a/testing/embedding/add1-test.c
+++ b/testing/embedding/add1-test.c
@@ -1,5 +1,9 @@
 #include <stdio.h>
 
+#ifdef _MSC_VER
+#include <windows.h>
+#endif
+
 extern int add1(int, int);
 
 
@@ -9,5 +13,9 @@
     x = add1(40, 2);
     y = add1(100, -5);
     printf("got: %d %d\n", x, y);
+#ifdef _MSC_VER
+    if (x == 0 && y == 0)
+        Sleep(2000);
+#endif
     return 0;
 }

From pypy.commits at gmail.com  Fri Jun 16 13:45:45 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 16 Jun 2017 10:45:45 -0700 (PDT)
Subject: [pypy-commit] pypy multiphase: really fix translation
Message-ID: <59441949.889bdf0a.4f78a.23af@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: multiphase
Changeset: r91616:8bf0d6064d37
Date: 2017-06-16 18:45 +0100
http://bitbucket.org/pypy/pypy/changeset/8bf0d6064d37/

Log:	really fix translation

diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py
--- a/pypy/module/cpyext/typeobject.py
+++ b/pypy/module/cpyext/typeobject.py
@@ -927,11 +927,12 @@
         i = 0
         while True:
             slotdef = slotdefs[i]
-            if slotdef.c_slot == 0:
+            slotnum = rffi.cast(lltype.Signed, slotdef.c_slot)
+            if slotnum == 0:
                 break
-            if slotdef.c_slot == cts.macros['Py_tp_base']:
+            elif slotnum == cts.macros['Py_tp_base']:
                 w_base = from_ref(space, cts.cast('PyObject*', slotdef.c_pfunc))
-            elif slotdef.c_slot == cts.macros['Py_tp_bases']:
+            elif slotnum == cts.macros['Py_tp_bases']:
                 bases = cts.cast('PyObject*', slotdef.c_pfunc)
                 bases_w = space.fixedview(from_ref(space, bases))
             i += 1
@@ -959,9 +960,9 @@
     i = 0
     while True:
         slotdef = slotdefs[i]
-        if slotdef.c_slot == 0:
+        slot = rffi.cast(lltype.Signed, slotdef.c_slot)
+        if slot == 0:
             break
-        slot = slotdef.c_slot
         if slot < 0:  # or slot > len(slotoffsets):
             raise oefmt(space.w_RuntimeError, "invalid slot offset")
         if slot in (cts.macros['Py_tp_base'], cts.macros['Py_tp_bases']):

From pypy.commits at gmail.com  Fri Jun 16 14:52:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 16 Jun 2017 11:52:16 -0700 (PDT)
Subject: [pypy-commit] cffi default: Issue #316
Message-ID: <594428e0.0f911c0a.5680c.ff71@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2981:cace5cac5ccb
Date: 2017-06-16 20:45 +0200
http://bitbucket.org/cffi/cffi/changeset/cace5cac5ccb/

Log:	Issue #316

	Call MessageBox() more generally on most stderr outputs from cffi

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -288,6 +288,8 @@
 
 #include "wchar_helper.h"
 
+#include "../cffi/_cffi_errors.h"
+
 typedef struct _cffi_allocator_s {
     PyObject *ca_alloc, *ca_free;
     int ca_dont_clear;
@@ -1829,7 +1831,8 @@
 }
 
 /* forward */
-static void _my_PyErr_WriteUnraisable(char *objdescr, PyObject *obj,
+static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb,
+                                      char *objdescr, PyObject *obj,
                                       char *extra_error_line);
 
 
@@ -1849,8 +1852,12 @@
             Py_DECREF(result);
         }
         else {
-            _my_PyErr_WriteUnraisable("From callback for ffi.gc ",
+            PyObject *ecap, *t, *v, *tb;
+            PyErr_Fetch(&t, &v, &tb);
+            ecap = _cffi_start_error_capture();
+            _my_PyErr_WriteUnraisable(t, v, tb, "From callback for ffi.gc ",
                                       origobj, NULL);
+            _cffi_stop_error_capture(ecap);
         }
         Py_DECREF(destructor);
 
@@ -5500,12 +5507,12 @@
     return convert_from_object(result, ctype, pyobj);
 }
 
-static void _my_PyErr_WriteUnraisable(char *objdescr, PyObject *obj,
+static void _my_PyErr_WriteUnraisable(PyObject *t, PyObject *v, PyObject *tb,
+                                      char *objdescr, PyObject *obj,
                                       char *extra_error_line)
 {
     /* like PyErr_WriteUnraisable(), but write a full traceback */
-    PyObject *f, *t, *v, *tb;
-    PyErr_Fetch(&t, &v, &tb);
+    PyObject *f;
 #if PY_MAJOR_VERSION >= 3
     /* jump through hoops to ensure the tb is attached to v, on Python 3 */
     PyErr_NormalizeException(&t, &v, &tb);
@@ -5595,8 +5602,12 @@
     }
     onerror_cb = PyTuple_GET_ITEM(cb_args, 3);
     if (onerror_cb == Py_None) {
-        _my_PyErr_WriteUnraisable("From cffi callback ", py_ob,
+        PyObject *ecap, *t, *v, *tb;
+        PyErr_Fetch(&t, &v, &tb);
+        ecap = _cffi_start_error_capture();
+        _my_PyErr_WriteUnraisable(t, v, tb, "From cffi callback ", py_ob,
                                   extra_error_line);
+        _cffi_stop_error_capture(ecap);
     }
     else {
         PyObject *exc1, *val1, *tb1, *res1, *exc2, *val2, *tb2;
@@ -5620,14 +5631,17 @@
         }
         else {
             /* double exception! print a double-traceback... */
+            PyObject *ecap;
             PyErr_Fetch(&exc2, &val2, &tb2);
-            PyErr_Restore(exc1, val1, tb1);
-            _my_PyErr_WriteUnraisable("From cffi callback ", py_ob,
+            ecap = _cffi_start_error_capture();
+            _my_PyErr_WriteUnraisable(exc1, val1, tb1,
+                                      "From cffi callback ", py_ob,
                                       extra_error_line);
-            PyErr_Restore(exc2, val2, tb2);
             extra_error_line = ("\nDuring the call to 'onerror', "
                                 "another exception occurred:\n\n");
-            _my_PyErr_WriteUnraisable(NULL, NULL, extra_error_line);
+            _my_PyErr_WriteUnraisable(exc2, val2, tb2,
+                                      NULL, NULL, extra_error_line);
+            _cffi_stop_error_capture(ecap);
         }
     }
     goto done;
diff --git a/cffi/_cffi_errors.h b/cffi/_cffi_errors.h
new file mode 100644
--- /dev/null
+++ b/cffi/_cffi_errors.h
@@ -0,0 +1,141 @@
+#ifndef CFFI_MESSAGEBOX
+# ifdef _MSC_VER
+#  define CFFI_MESSAGEBOX  1
+# else
+#  define CFFI_MESSAGEBOX  0
+# endif
+#endif
+
+
+#if CFFI_MESSAGEBOX
+/* Windows only: logic to take the Python-CFFI embedding logic
+   initialization errors and display them in a background thread
+   with MessageBox.  The idea is that if the whole program closes
+   as a result of this problem, then likely it is already a console
+   program and you can read the stderr output in the console too.
+   If it is not a console program, then it will likely show its own
+   dialog to complain, or generally not abruptly close, and for this
+   case the background thread should stay alive.
+*/
+static void *volatile _cffi_bootstrap_text;
+
+static PyObject *_cffi_start_error_capture(void)
+{
+    PyObject *result = NULL;
+    PyObject *x, *m, *bi;
+
+    if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+            (void *)1, NULL) != NULL)
+        return (PyObject *)1;
+
+    m = PyImport_AddModule("_cffi_error_capture");
+    if (m == NULL)
+        goto error;
+
+    result = PyModule_GetDict(m);
+    if (result == NULL)
+        goto error;
+
+    bi = PyImport_ImportModule("__builtin__");
+    if (bi == NULL)
+        goto error;
+    PyDict_SetItemString(result, "__builtins__", bi);
+    Py_DECREF(bi);
+
+    x = PyRun_String(
+        "import sys\n"
+        "class FileLike:\n"
+        "  def write(self, x):\n"
+        "    of.write(x)\n"
+        "    self.buf += x\n"
+        "fl = FileLike()\n"
+        "fl.buf = ''\n"
+        "of = sys.stderr\n"
+        "sys.stderr = fl\n"
+        "def done():\n"
+        "  sys.stderr = of\n"
+        "  return fl.buf\n",   /* make sure the returned value stays alive */
+        Py_file_input,
+        result, result);
+    Py_XDECREF(x);
+
+ error:
+    if (PyErr_Occurred())
+    {
+        PyErr_WriteUnraisable(Py_None);
+        PyErr_Clear();
+    }
+    return result;
+}
+
+#pragma comment(lib, "user32.lib")
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+#if PY_MAJOR_VERSION >= 3
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#else
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#endif
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+static void _cffi_stop_error_capture(PyObject *ecap)
+{
+    PyObject *s;
+    void *text;
+
+    if (ecap == (PyObject *)1)
+        return;
+
+    if (ecap == NULL)
+        goto error;
+
+    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
+    if (s == NULL)
+        goto error;
+
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+#if PY_MAJOR_VERSION >= 3
+    text = PyUnicode_AsWideCharString(s, NULL);
+#else
+    text = PyString_AsString(s);
+#endif
+
+    _cffi_bootstrap_text = text;
+
+    if (text != NULL)
+    {
+        HANDLE h;
+        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                         NULL, 0, NULL);
+        if (h != NULL)
+            CloseHandle(h);
+    }
+    /* decref the string, but it should stay alive as 'fl.buf'
+       in the small module above.  It will really be freed only if
+       we later get another similar error.  So it's a leak of at
+       most one copy of the small module.  That's fine for this
+       situation which is usually a "fatal error" anyway. */
+    Py_DECREF(s);
+    PyErr_Clear();
+    return;
+
+  error:
+    _cffi_bootstrap_text = NULL;
+    PyErr_Clear();
+}
+
+#else
+
+static PyObject *_cffi_start_error_capture(void) { return NULL; }
+static void _cffi_stop_error_capture(PyObject *ecap) { }
+
+#endif
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
--- a/cffi/_embedding.h
+++ b/cffi/_embedding.h
@@ -109,132 +109,7 @@
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
-
-#ifdef _MSC_VER
-/* Windows only: logic to take the Python-CFFI embedding logic
-   initialization errors and display them in a background thread
-   with MessageBox.  The idea is that if the whole program closes
-   as a result of this problem, then likely it is already a console
-   program and you can read the stderr output in the console too.
-   If it is not a console program, then it will likely show its own
-   dialog to complain, or generally not abruptly close, and for this
-   case the background thread should stay alive.
-*/
-static PyObject *_cffi_bootstrap_string;
-static void *volatile _cffi_bootstrap_text;
-
-static PyObject *_cffi_start_error_capture(void)
-{
-    PyObject *result = NULL;
-    PyObject *x, *m, *bi;
-
-    if (!cffi_compare_and_swap(&_cffi_bootstrap_text, NULL, (void *)1))
-        return (PyObject *)1;
-
-    m = PyImport_AddModule("_cffi_error_capture");
-    if (m == NULL)
-        goto error;
-
-    result = PyModule_GetDict(m);
-    if (result == NULL)
-        goto error;
-
-    bi = PyImport_ImportModule("__builtin__");
-    if (bi == NULL)
-        goto error;
-    PyDict_SetItemString(result, "__builtins__", bi);
-    Py_DECREF(bi);
-
-    x = PyRun_String(
-        "import sys\n"
-        "class FileLike:\n"
-        "  def write(self, x):\n"
-        "    sys.__stderr__.write(x)\n"
-        "    self.buf += x\n"
-        "fl = FileLike()\n"
-        "fl.buf = ''\n"
-        "sys.stderr = fl\n"
-        "def done():\n"
-        "  sys.stderr = sys.__stderr__\n"
-        "  return fl.buf\n",   /* make sure the returned value stays alive */
-        Py_file_input,
-        result, result);
-    Py_XDECREF(x);
-
- error:
-    if (PyErr_Occurred())
-    {
-        PyErr_WriteUnraisable(Py_None);
-        PyErr_Clear();
-    }
-    return result;
-}
-
-#pragma comment(lib, "user32.lib")
-
-static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
-{
-    Sleep(666);    /* may be interrupted if the whole process is closing */
-#if PY_MAJOR_VERSION >= 3
-    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
-                L"Python-CFFI embedding error",
-                MB_OK | MB_ICONERROR);
-#else
-    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
-                "Python-CFFI embedding error",
-                MB_OK | MB_ICONERROR);
-#endif
-    _cffi_bootstrap_text = NULL;
-    return 0;
-}
-
-static void _cffi_stop_error_capture(PyObject *ecap)
-{
-    PyObject *s;
-    void *text;
-
-    if (ecap == (PyObject *)1)
-        return;
-
-    if (ecap == NULL)
-        goto error;
-
-    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
-    if (s == NULL)
-        goto error;
-
-    /* Show a dialog box, but in a background thread, and
-       never show multiple dialog boxes at once. */
-#if PY_MAJOR_VERSION >= 3
-    text = PyUnicode_AsWideCharString(s, NULL);
-#else
-    text = PyString_AsString(s);
-#endif
-
-    _cffi_bootstrap_text = text;
-
-    if (text != NULL)
-    {
-        HANDLE h;
-        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
-                         NULL, 0, NULL);
-        if (h != NULL)
-            CloseHandle(h);
-    }
-    /* decref the string, but it should stay alive as 'fl.buf'
-       in the small module above.  It will really be freed only if
-       we later get another similar error.  So it's a leak of at
-       most one copy of the small module.  That's fine for this
-       situation which is usually a "fatal error" anyway. */
-    Py_DECREF(s);
-    PyErr_Clear();
-    return;
-
-  error:
-    _cffi_bootstrap_text = NULL;
-    PyErr_Clear();
-}
-#endif
+#include "_cffi_errors.h"
 
 
 #define _cffi_call_python_org  _cffi_exports[_CFFI_CPIDX]
@@ -347,14 +222,10 @@
         /* Print as much information as potentially useful.
            Debugging load-time failures with embedding is not fun
         */
-#ifdef _MSC_VER
         PyObject *ecap;
-#endif
         PyObject *exception, *v, *tb, *f, *modules, *mod;
         PyErr_Fetch(&exception, &v, &tb);
-#ifdef _MSC_VER
         ecap = _cffi_start_error_capture();
-#endif
         f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString(
@@ -387,9 +258,7 @@
             PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
             PyFile_WriteString("\n\n", f);
         }
-#ifdef _MSC_VER
         _cffi_stop_error_capture(ecap);
-#endif
     }
     result = -1;
     goto done;
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -308,6 +308,8 @@
                 base_module_name,))
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
+            i = lines.index('#include "_cffi_errors.h"\n')
+            lines[i:i+1] = self._rel_readlines('_cffi_errors.h')
             prnt(''.join(lines))
             self.needs_version(VERSION_EMBEDDED)
         #
diff --git a/doc/source/whatsnew.rst b/doc/source/whatsnew.rst
--- a/doc/source/whatsnew.rst
+++ b/doc/source/whatsnew.rst
@@ -26,6 +26,18 @@
   cffi.FFI()`` at module level just after ``import cffi``; and in
   out-of-line mode you don't instantiate ``FFI`` explicitly at all.)
 
+* Windows: using callbacks can be messy because the CFFI internal error
+  messages show up to stderr---but stderr goes nowhere in many
+  applications.  This makes it particularly hard to get started with the
+  embedding mode.  (Once you get started, you can at least use
+  ``@ffi.def_extern(onerror=...)`` and send the error logs where it
+  makes sense for your application, or record them in log files, and so
+  on.)  So what is new in CFFI is that now, on Windows CFFI will try to
+  open a non-modal MessageBox (in addition to sending raw messages to
+  stderr).  The MessageBox is only visible if the process stays alive:
+  typically, console applications that crash close immediately, but that
+  is also the situation where stderr should be visible anyway.
+
 
 v1.10.1
 =======

From pypy.commits at gmail.com  Sat Jun 17 12:27:34 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:27:34 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fixes for test_descr
Message-ID: <59455876.a198df0a.337cc.988f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91617:70d5dcb50915
Date: 2017-06-17 16:01 +0200
http://bitbucket.org/pypy/pypy/changeset/70d5dcb50915/

Log:	Fixes for test_descr

diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -1663,7 +1663,8 @@
         self.assertEqual(b.foo, 3)
         self.assertEqual(b.__class__, D)
 
-    @unittest.expectedFailure
+    #@unittest.expectedFailure --- on CPython.  On PyPy, the test passes
+    @support.impl_detail(cpython=False)
     def test_bad_new(self):
         self.assertRaises(TypeError, object.__new__)
         self.assertRaises(TypeError, object.__new__, '')
diff --git a/pypy/objspace/std/test/test_typeobject.py b/pypy/objspace/std/test/test_typeobject.py
--- a/pypy/objspace/std/test/test_typeobject.py
+++ b/pypy/objspace/std/test/test_typeobject.py
@@ -477,7 +477,8 @@
             a = 1
 
         class mymeta(type):
-            def mro(self):
+            def mro(self, ignore=False):
+                assert ignore or self.__mro__ is None
                 return [self, object]
 
         class B_mro(A_mro, metaclass=mymeta):
@@ -485,7 +486,7 @@
 
         assert B_mro.__bases__ == (A_mro,)
         assert B_mro.__mro__ == (B_mro, object)
-        assert B_mro.mro() == [B_mro, object]
+        assert B_mro.mro(ignore=True) == [B_mro, object]
         assert B_mro.b == 1
         assert B_mro().b == 1
         assert getattr(B_mro, 'a', None) == None
diff --git a/pypy/objspace/std/typeobject.py b/pypy/objspace/std/typeobject.py
--- a/pypy/objspace/std/typeobject.py
+++ b/pypy/objspace/std/typeobject.py
@@ -810,7 +810,10 @@
 
 def descr_get__mro__(space, w_type):
     w_type = _check(space, w_type)
-    return space.newtuple(w_type.mro_w)
+    if w_type.hasmro:
+        return space.newtuple(w_type.mro_w)
+    else:
+        return space.w_None
 
 def descr_mro(space, w_type):
     """Return a type's method resolution order."""
@@ -1227,6 +1230,7 @@
         ensure_module_attr(w_self)
     ensure_hash(w_self)
     w_self.mro_w = []      # temporarily
+    w_self.hasmro = False
     compute_mro(w_self)
 
 def ensure_static_new(w_self):
@@ -1264,8 +1268,10 @@
             w_mro = space.call_function(w_mro_meth)
             mro_w = space.fixedview(w_mro)
             w_self.mro_w = validate_custom_mro(space, mro_w)
+            w_self.hasmro = True
             return    # done
     w_self.mro_w = w_self.compute_default_mro()[:]
+    w_self.hasmro = True
 
 def validate_custom_mro(space, mro_w):
     # do some checking here.  Note that unlike CPython, strange MROs

From pypy.commits at gmail.com  Sat Jun 17 12:27:36 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:27:36 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add a failing test. Unsure how to fix
 it---I can't quite follow the mess
Message-ID: <59455878.04c41c0a.995f9.6a9c@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91618:9d66d935ba76
Date: 2017-06-17 16:36 +0200
http://bitbucket.org/pypy/pypy/changeset/9d66d935ba76/

Log:	Add a failing test. Unsure how to fix it---I can't quite follow the
	mess of buffers and memoryviews for now...

diff --git a/pypy/module/_io/test/test_fileio.py b/pypy/module/_io/test/test_fileio.py
--- a/pypy/module/_io/test/test_fileio.py
+++ b/pypy/module/_io/test/test_fileio.py
@@ -5,7 +5,8 @@
 
 
 class AppTestFileIO:
-    spaceconfig = dict(usemodules=['_io'] + (['fcntl'] if os.name != 'nt' else []))
+    spaceconfig = dict(usemodules=['_io', 'array'] +
+                                  (['fcntl'] if os.name != 'nt' else []))
 
     def setup_method(self, meth):
         tmpfile = udir.join('tmpfile')
@@ -188,6 +189,14 @@
         assert f.readinto(a) == 1000
         assert a == b'a' * 1000 + b'x' * 24
 
+    def test_readinto_array(self):
+        import _io, array
+        buffer = array.array('i', [0]*10)
+        m = memoryview(buffer)
+        f = _io.FileIO(self.tmpfile, 'r+')
+        assert f.readinto(m[1:9]) == 5
+        assert buffer[1] in (0x610a620a, 0x0a620a61)
+
     def test_nonblocking_read(self):
         try:
             import os, fcntl

From pypy.commits at gmail.com  Sat Jun 17 12:27:38 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:27:38 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix test
Message-ID: <5945587a.87e51c0a.345c.594d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91619:19f65d747b8e
Date: 2017-06-17 16:41 +0200
http://bitbucket.org/pypy/pypy/changeset/19f65d747b8e/

Log:	Fix test

diff --git a/pypy/module/pypyjit/test_pypy_c/test_weakref.py b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
--- a/pypy/module/pypyjit/test_pypy_c/test_weakref.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_weakref.py
@@ -19,12 +19,15 @@
         """, [500])
         loop, = log.loops_by_filename(self.filepath)
         assert loop.match("""
-        i58 = getfield_gc_i(p18, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current .>)
+        i58 = getfield_gc_i(p18, descr=<FieldS pypy.module.__builtin__.functional.W_IntRangeIterator.inst_current .>)
         i60 = int_lt(i58, i31)
         guard_true(i60, descr=...)
         i61 = int_add(i58, 1)
-        setfield_gc(p18, i61, descr=<FieldS pypy.module.__builtin__.functional.W_XRangeIterator.inst_current 8>)
+        dummy_get_utf8?
+        setfield_gc(p18, i61, descr=<FieldS pypy.module.__builtin__.functional.W_IntRangeIterator.inst_current 8>)
         guard_not_invalidated(descr=...)
+        dummy_get_utf8?
+        dummy_get_utf8?
         p65 = getfield_gc_r(p14, descr=<FieldP .+inst_map \d+>)
         guard_value(p65, ConstPtr(ptr45), descr=...)
         p66 = getfield_gc_r(p14, descr=<FieldP .+inst__value0 \d+>)

From pypy.commits at gmail.com  Sat Jun 17 12:27:39 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:27:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Improve test,
 now not passing for maybe a good reason
Message-ID: <5945587b.423f1c0a.5d5a4.cb13@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91620:3351108873e1
Date: 2017-06-17 18:26 +0200
http://bitbucket.org/pypy/pypy/changeset/3351108873e1/

Log:	Improve test, now not passing for maybe a good reason

diff --git a/pypy/module/pypyjit/test_pypy_c/test_string.py b/pypy/module/pypyjit/test_pypy_c/test_string.py
--- a/pypy/module/pypyjit/test_pypy_c/test_string.py
+++ b/pypy/module/pypyjit/test_pypy_c/test_string.py
@@ -65,25 +65,13 @@
             i97 = int_ge(i94, i53)
             guard_false(i97, descr=...)
             i98 = strgetitem(p52, i94)
-            p101 = force_token()
             p103 = newstr(1)
             strsetitem(p103, 0, i98)
-            p104 = new(descr=<SizeDescr ..?>)
-            p106 = newunicode(1)
-            setfield_gc(p0, p101, descr=<FieldP pypy.interpreter.pyframe.PyFrame.vable_token .>)
-            setfield_gc(p104, p106, descr=<FieldP unicodebuilder.current_buf ..?>)
-            setfield_gc(p104, 0, descr=<FieldS unicodebuilder.current_pos ..?>)
-            setfield_gc(p104, 1, descr=<FieldS unicodebuilder.current_end ..?>)
-            setfield_gc(p104, 1, descr=<FieldS unicodebuilder.total_size 32>)
-            i113 = call_may_force_i(ConstClass(str_decode_utf_8_impl), p103, 1, ConstPtr(null), 1, 0, 0, p104, descr=<Calli . ririiir EF=7>)
-            guard_not_forced(descr=...)
+            p296 = call_r(ConstClass(str_decode_utf_8), p103, 1, ConstPtr(null), 1, ConstClass(raise_unicode_exception_decode), 0, descr=<Callr . ririii EF=4>)
             guard_no_exception(descr=...)
-            p116 = call_r(ConstClass(ll_build_trampoline__), p104, descr=<Callr . r EF=5>)
-            guard_no_exception(descr=...)
-            guard_nonnull(p116, descr=...)
-            guard_not_invalidated(descr=...)
-            i119 = int_ge(i94, i46)
-            guard_false(i119, descr=...)
+            p116 = getfield_gc_r(p296, descr=<FieldP tuple2.item0 . pure>)
+            i99 = int_ge(i94, i46)
+            guard_false(i99, descr=...)
             i120 = unicodegetitem(p45, i94)
             i122 = call_i(ConstClass(_ll_2_str_eq_nonnull_char__rpy_unicodePtr_UniChar), p116, i120, descr=<Calli . ri EF=0 OS=49>)
             guard_true(i122, descr=...)

From pypy.commits at gmail.com  Sat Jun 17 12:29:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:29:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: Add "can_be_None=False" in the result
 annotation of these methods
Message-ID: <594558fd.94da1c0a.1d28d.f73f@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91621:2e1b4bdca21f
Date: 2017-06-17 18:29 +0200
http://bitbucket.org/pypy/pypy/changeset/2e1b4bdca21f/

Log:	Add "can_be_None=False" in the result annotation of these methods

diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -636,7 +636,7 @@
         return SomeInteger(nonneg=True)
 
     def method_build(self):
-        return SomeString()
+        return SomeString(can_be_None=False)
 
     def rtyper_makerepr(self, rtyper):
         from rpython.rtyper.lltypesystem.rbuilder import stringbuilder_repr
@@ -676,7 +676,7 @@
         return SomeInteger(nonneg=True)
 
     def method_build(self):
-        return SomeUnicodeString()
+        return SomeUnicodeString(can_be_None=False)
 
     def rtyper_makerepr(self, rtyper):
         from rpython.rtyper.lltypesystem.rbuilder import unicodebuilder_repr

From pypy.commits at gmail.com  Sat Jun 17 12:33:11 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 17 Jun 2017 09:33:11 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <594559c7.4e331c0a.78f55.e649@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91622:d7d9692a9f74
Date: 2017-06-17 18:32 +0200
http://bitbucket.org/pypy/pypy/changeset/d7d9692a9f74/

Log:	hg merge default

diff --git a/.hgignore b/.hgignore
--- a/.hgignore
+++ b/.hgignore
@@ -1,6 +1,6 @@
 syntax: glob
 *.py[co]
-*.sw[po]
+*.sw[pon]
 *~
 .*.swp
 .idea
@@ -10,6 +10,8 @@
 .venv
 .cache
 
+.cache/
+.gdb_history
 syntax: regexp
 ^testresult$
 ^site-packages$
@@ -90,7 +92,6 @@
 .hypothesis/
 ^release/
 ^rpython/_cache$
-^\.cache$
 
 pypy/module/cppyy/.+/*\.pcm
 
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -93,7 +93,8 @@
     libsqlite3
 
 curses
-    libncurses
+    libncurses-dev   (for PyPy2)
+    libncursesw-dev  (for PyPy3)
 
 gdbm
     libgdbm-dev
@@ -106,12 +107,13 @@
 
 To run untranslated tests, you need the Boehm garbage collector libgc.
 
-On Debian, this is the command to install all build-time dependencies::
+On Debian and Ubuntu, this is the command to install all build-time
+dependencies::
 
     apt-get install gcc make libffi-dev pkg-config libz-dev libbz2-dev \
     libsqlite3-dev libncurses-dev libexpat1-dev libssl-dev libgdbm-dev \
     tk-dev libgc-dev python-cffi \
-    liblzma-dev  # For lzma on PyPy3.
+    liblzma-dev libncursesw-dev      # these two only needed on PyPy3
 
 On Fedora::
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -9,3 +9,11 @@
 .. branch: cffi-char16-char32
 
 The two ``cffi-*`` branches are part of the upgrade to cffi 1.11.
+
+.. branch: ctypes_char_indexing
+
+Indexing into char* behaves differently than CPython
+
+.. branch: vmprof-0.4.8
+
+Improve and fix issues with vmprof
diff --git a/pypy/module/cpyext/setobject.py b/pypy/module/cpyext/setobject.py
--- a/pypy/module/cpyext/setobject.py
+++ b/pypy/module/cpyext/setobject.py
@@ -9,6 +9,7 @@
 
 
 PySet_Check, PySet_CheckExact = build_type_checkers("Set")
+PyFrozenSet_Check, PyFrozenSet_CheckExact = build_type_checkers("FrozenSet")
 
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def PyAnySet_Check(space, w_obj):
@@ -91,7 +92,7 @@
     """Return the length of a set or frozenset object. Equivalent to
     len(anyset).  Raises a PyExc_SystemError if anyset is not a set, frozenset,
     or an instance of a subtype."""
-    if not PySet_Check(space, ref):
+    if not PyAnySet_Check(space, ref):
         raise oefmt(space.w_TypeError, "expected set object")
     return PySet_GET_SIZE(space, ref)
 
@@ -104,3 +105,20 @@
     set, frozenset, or an instance of a subtype."""
     w_res = space.contains(w_obj, w_key)
     return space.int_w(w_res)
+
+ at cpython_api([PyObject], PyObject)
+def PyFrozenSet_New(space, w_iterable):
+    """Return a new frozenset containing objects returned by the iterable.
+    The iterable may be NULL to create a new empty frozenset.  Return the new
+    set on success or NULL on failure.  Raise TypeError if iterable is
+    not actually iterable.
+
+    Now guaranteed to return a brand-new frozenset.  Formerly,
+    frozensets of zero-length were a singleton.  This got in the way of
+    building-up new frozensets with PySet_Add()."""
+    if w_iterable is None:
+        return space.call_function(space.w_frozenset)
+    else:
+        return space.call_function(space.w_frozenset, w_iterable)
+
+
diff --git a/pypy/module/cpyext/stubs.py b/pypy/module/cpyext/stubs.py
--- a/pypy/module/cpyext/stubs.py
+++ b/pypy/module/cpyext/stubs.py
@@ -1408,26 +1408,6 @@
     equivalent to the Python expression o.count(value)."""
     raise NotImplementedError
 
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyFrozenSet_Check(space, p):
-    """Return true if p is a frozenset object or an instance of a
-    subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
-def PyFrozenSet_CheckExact(space, p):
-    """Return true if p is a frozenset object but not an instance of a
-    subtype."""
-    raise NotImplementedError
-
- at cpython_api([PyObject], PyObject)
-def PyFrozenSet_New(space, iterable):
-    """Return a new frozenset containing objects returned by the iterable.
-    The iterable may be NULL to create a new empty frozenset.  Return the new
-    set on success or NULL on failure.  Raise TypeError if iterable is
-    not actually iterable."""
-    raise NotImplementedError
-
 @cpython_api([FILE, rffi.CCHARP], rffi.INT_real, error=-1)
 def Py_FdIsInteractive(space, fp, filename):
     """Return true (nonzero) if the standard I/O file fp with name filename is
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -9,9 +9,11 @@
 class TestTupleObject(BaseApiTest):
     def test_setobj(self, space, api):
         assert not api.PySet_Check(space.w_None)
+        assert not api.PyFrozenSet_Check(space.w_None)
         assert api.PySet_Add(space.w_None, space.w_None) == -1
         api.PyErr_Clear()
         w_set = space.call_function(space.w_set)
+        assert not api.PyFrozenSet_CheckExact(w_set)
         space.call_method(w_set, 'update', space.wrap([1,2,3,4]))
         assert api.PySet_Size(w_set) == 4
         assert api.PySet_GET_SIZE(w_set) == 4
@@ -21,6 +23,8 @@
     def test_set_add_discard(self, space, api):
         w_set = api.PySet_New(None)
         assert api.PySet_Size(w_set) == 0
+        w_set = api.PyFrozenSet_New(space.wrap([1,2,3,4]))
+        assert api.PySet_Size(w_set) == 4
         w_set = api.PySet_New(space.wrap([1,2,3,4]))
         assert api.PySet_Size(w_set) == 4
         api.PySet_Add(w_set, space.wrap(6))
diff --git a/rpython/jit/backend/x86/assembler.py b/rpython/jit/backend/x86/assembler.py
--- a/rpython/jit/backend/x86/assembler.py
+++ b/rpython/jit/backend/x86/assembler.py
@@ -980,7 +980,8 @@
         from rpython.rlib.rvmprof.rvmprof import cintf
         # edx = address of pypy_threadlocal_s
         self.mc.MOV_rs(edx.value, THREADLOCAL_OFS)
-        self.mc.AND_ri(edx.value, ~1)
+        if self._is_asmgcc():
+            self.mc.AND_ri(edx.value, ~1)
         # eax = (our local vmprof_tl_stack).next
         self.mc.MOV_rs(eax.value, (FRAME_FIXED_SIZE - 4 + 0) * WORD)
         # save in vmprof_tl_stack the value eax
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -636,7 +636,7 @@
         return SomeInteger(nonneg=True)
 
     def method_build(self):
-        return SomeString()
+        return SomeString(can_be_None=False)
 
     def rtyper_makerepr(self, rtyper):
         from rpython.rtyper.lltypesystem.rbuilder import stringbuilder_repr
@@ -676,7 +676,7 @@
         return SomeInteger(nonneg=True)
 
     def method_build(self):
-        return SomeUnicodeString()
+        return SomeUnicodeString(can_be_None=False)
 
     def rtyper_makerepr(self, rtyper):
         from rpython.rtyper.lltypesystem.rbuilder import unicodebuilder_repr
diff --git a/rpython/translator/platform/__init__.py b/rpython/translator/platform/__init__.py
--- a/rpython/translator/platform/__init__.py
+++ b/rpython/translator/platform/__init__.py
@@ -260,7 +260,8 @@
     # Only required on armhf and mips{,el}, not armel. But there's no way to
     # detect armhf without shelling out
     if (platform.architecture()[0] == '64bit'
-            or platform.machine().startswith(('arm', 'mips', 'ppc'))):
+            or platform.machine().startswith(
+                ('arm', 'm68k', 'mips', 'parisc', 'ppc', 'sh4'))):
         host_factory = LinuxPIC
     else:
         host_factory = Linux

From pypy.commits at gmail.com  Sat Jun 17 21:43:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sat, 17 Jun 2017 18:43:43 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix sys.builtin_module_names to actually
 list all the built-in modules
Message-ID: <5945dacf.49911c0a.5be05.2fd9@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91623:12adced6b2f7
Date: 2017-06-18 02:43 +0100
http://bitbucket.org/pypy/pypy/changeset/12adced6b2f7/

Log:	Fix sys.builtin_module_names to actually list all the built-in
	modules and only them, which allows importlib to correctly set
	__loader__ and __spec__ for them.

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -546,14 +546,6 @@
         self._builtinmodule_list = modules
         return self._builtinmodule_list
 
-    ALL_BUILTIN_MODULES = [
-        'posix', 'nt', 'os2', 'mac', 'ce', 'riscos',
-        'math', 'array', 'select',
-        '_random', '_sre', 'time', '_socket', 'errno',
-        'unicodedata',
-        'parser', 'fcntl', '_codecs', 'binascii'
-    ]
-
     @not_rpython
     def make_builtins(self):
         "only for initializing the space."
@@ -580,11 +572,6 @@
         w_builtin.install()
         self.setitem(self.builtin.w_dict, self.newtext('__builtins__'), w_builtin)
 
-        # exceptions was bootstrapped as '__exceptions__' but still
-        # lives in pypy/module/exceptions, we rename it below for
-        # sys.builtin_module_names
-        bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
-        installed_builtin_modules = list(bootstrap_modules)
 
         exception_types_w = self.export_builtin_exceptions()
 
@@ -595,15 +582,13 @@
             self.setitem(self.builtin.w_dict, self.newtext(name), w_type)
 
         # install mixed modules
+        bootstrap_modules = set(('sys', 'imp', 'builtins', 'exceptions'))
         for mixedname in self.get_builtinmodule_to_install():
             if mixedname not in bootstrap_modules:
-                self.install_mixedmodule(mixedname, installed_builtin_modules)
+                self.install_mixedmodule(mixedname)
 
-        installed_builtin_modules.remove('exceptions')
-        installed_builtin_modules.append('__exceptions__')
-        installed_builtin_modules.sort()
         w_builtin_module_names = self.newtuple(
-            [self.newtext(fn) for fn in installed_builtin_modules])
+            [self.newtext(name) for name in sorted(self.builtin_modules)])
 
         # force this value into the dict without unlazyfying everything
         self.setitem(self.sys.w_dict, self.newtext('builtin_module_names'),
@@ -635,13 +620,8 @@
         return exc_types_w
 
     @not_rpython
-    def install_mixedmodule(self, mixedname, installed_builtin_modules):
-        modname = self.setbuiltinmodule(mixedname)
-        if modname:
-            assert modname not in installed_builtin_modules, (
-                "duplicate interp-level module enabled for the "
-                "app-level module %r" % (modname,))
-            installed_builtin_modules.append(modname)
+    def install_mixedmodule(self, mixedname):
+        self.setbuiltinmodule(mixedname)
 
     @not_rpython
     def setup_builtin_modules(self):
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -38,6 +38,10 @@
     def install(self):
         """NOT_RPYTHON: installs this module into space.builtin_modules"""
         modulename = self.space.text0_w(self.w_name)
+        if modulename in self.space.builtin_modules:
+            raise ValueError(
+                "duplicate interp-level module enabled for the "
+                "app-level module %r" % (modulename,))
         self.space.builtin_modules[modulename] = self
 
     def setup_after_space_initialization(self):
diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -108,6 +108,13 @@
         assert not imp.is_builtin('hello.world.this.is.never.a.builtin.module.name')
         assert not imp.is_frozen('hello.world.this.is.never.a.frozen.module.name')
 
+    def test_is_builtin(self):
+        import sys, imp
+        for name in sys.builtin_module_names:
+            assert imp.is_builtin(name)
+            mod = imp.init_builtin(name)
+            assert mod
+            assert mod.__spec__
 
     def test_load_module_py(self):
         import imp

From pypy.commits at gmail.com  Mon Jun 19 03:37:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 00:37:42 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/cace5cac5ccb
Message-ID: <59477f46.162d1c0a.a4d57.1158@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91624:211e6e8190f1
Date: 2017-06-19 09:34 +0200
http://bitbucket.org/pypy/pypy/changeset/211e6e8190f1/

Log:	update to cffi/cace5cac5ccb

diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -0,0 +1,141 @@
+#ifndef CFFI_MESSAGEBOX
+# ifdef _MSC_VER
+#  define CFFI_MESSAGEBOX  1
+# else
+#  define CFFI_MESSAGEBOX  0
+# endif
+#endif
+
+
+#if CFFI_MESSAGEBOX
+/* Windows only: logic to take the Python-CFFI embedding logic
+   initialization errors and display them in a background thread
+   with MessageBox.  The idea is that if the whole program closes
+   as a result of this problem, then likely it is already a console
+   program and you can read the stderr output in the console too.
+   If it is not a console program, then it will likely show its own
+   dialog to complain, or generally not abruptly close, and for this
+   case the background thread should stay alive.
+*/
+static void *volatile _cffi_bootstrap_text;
+
+static PyObject *_cffi_start_error_capture(void)
+{
+    PyObject *result = NULL;
+    PyObject *x, *m, *bi;
+
+    if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+            (void *)1, NULL) != NULL)
+        return (PyObject *)1;
+
+    m = PyImport_AddModule("_cffi_error_capture");
+    if (m == NULL)
+        goto error;
+
+    result = PyModule_GetDict(m);
+    if (result == NULL)
+        goto error;
+
+    bi = PyImport_ImportModule("__builtin__");
+    if (bi == NULL)
+        goto error;
+    PyDict_SetItemString(result, "__builtins__", bi);
+    Py_DECREF(bi);
+
+    x = PyRun_String(
+        "import sys\n"
+        "class FileLike:\n"
+        "  def write(self, x):\n"
+        "    of.write(x)\n"
+        "    self.buf += x\n"
+        "fl = FileLike()\n"
+        "fl.buf = ''\n"
+        "of = sys.stderr\n"
+        "sys.stderr = fl\n"
+        "def done():\n"
+        "  sys.stderr = of\n"
+        "  return fl.buf\n",   /* make sure the returned value stays alive */
+        Py_file_input,
+        result, result);
+    Py_XDECREF(x);
+
+ error:
+    if (PyErr_Occurred())
+    {
+        PyErr_WriteUnraisable(Py_None);
+        PyErr_Clear();
+    }
+    return result;
+}
+
+#pragma comment(lib, "user32.lib")
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+#if PY_MAJOR_VERSION >= 3
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#else
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#endif
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+static void _cffi_stop_error_capture(PyObject *ecap)
+{
+    PyObject *s;
+    void *text;
+
+    if (ecap == (PyObject *)1)
+        return;
+
+    if (ecap == NULL)
+        goto error;
+
+    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
+    if (s == NULL)
+        goto error;
+
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+#if PY_MAJOR_VERSION >= 3
+    text = PyUnicode_AsWideCharString(s, NULL);
+#else
+    text = PyString_AsString(s);
+#endif
+
+    _cffi_bootstrap_text = text;
+
+    if (text != NULL)
+    {
+        HANDLE h;
+        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                         NULL, 0, NULL);
+        if (h != NULL)
+            CloseHandle(h);
+    }
+    /* decref the string, but it should stay alive as 'fl.buf'
+       in the small module above.  It will really be freed only if
+       we later get another similar error.  So it's a leak of at
+       most one copy of the small module.  That's fine for this
+       situation which is usually a "fatal error" anyway. */
+    Py_DECREF(s);
+    PyErr_Clear();
+    return;
+
+  error:
+    _cffi_bootstrap_text = NULL;
+    PyErr_Clear();
+}
+
+#else
+
+static PyObject *_cffi_start_error_capture(void) { return NULL; }
+static void _cffi_stop_error_capture(PyObject *ecap) { }
+
+#endif
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -109,6 +109,8 @@
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
+#include "_cffi_errors.h"
+
 
 #define _cffi_call_python_org  _cffi_exports[_CFFI_CPIDX]
 
@@ -220,8 +222,16 @@
         /* Print as much information as potentially useful.
            Debugging load-time failures with embedding is not fun
         */
+        PyObject *ecap;
         PyObject *exception, *v, *tb, *f, *modules, *mod;
         PyErr_Fetch(&exception, &v, &tb);
+        ecap = _cffi_start_error_capture();
+        f = PySys_GetObject((char *)"stderr");
+        if (f != NULL && f != Py_None) {
+            PyFile_WriteString(
+                "Failed to initialize the Python-CFFI embedding logic:\n\n", f);
+        }
+
         if (exception != NULL) {
             PyErr_NormalizeException(&exception, &v, &tb);
             PyErr_Display(exception, v, tb);
@@ -230,7 +240,6 @@
         Py_XDECREF(v);
         Py_XDECREF(tb);
 
-        f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
                                "\ncompiled with cffi version: 1.11.0"
@@ -249,6 +258,7 @@
             PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
             PyFile_WriteString("\n\n", f);
         }
+        _cffi_stop_error_capture(ecap);
     }
     result = -1;
     goto done;
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -75,9 +75,10 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
+        self._typecache = model.get_typecache(backend)
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
-        for name in backend.__dict__:
+        for name in list(backend.__dict__):
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -568,22 +568,26 @@
 
 
 global_lock = allocate_lock()
+_typecache_cffi_backend = weakref.WeakValueDictionary()
+
+def get_typecache(backend):
+    # returns _typecache_cffi_backend if backend is the _cffi_backend
+    # module, or type(backend).__typecache if backend is an instance of
+    # CTypesBackend (or some FakeBackend class during tests)
+    if isinstance(backend, types.ModuleType):
+        return _typecache_cffi_backend
+    with global_lock:
+        if not hasattr(type(backend), '__typecache'):
+            type(backend).__typecache = weakref.WeakValueDictionary()
+        return type(backend).__typecache
 
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
     try:
-        return ffi._backend.__typecache[key]
+        return ffi._typecache[key]
     except KeyError:
         pass
-    except AttributeError:
-        # initialize the __typecache attribute, either at the module level
-        # if ffi._backend is a module, or at the class level if ffi._backend
-        # is some instance.
-        if isinstance(ffi._backend, types.ModuleType):
-            ffi._backend.__typecache = weakref.WeakValueDictionary()
-        else:
-            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
@@ -591,7 +595,7 @@
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
-    cache = ffi._backend.__typecache
+    cache = ffi._typecache
     with global_lock:
         res1 = cache.get(key)
         if res1 is None:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -308,6 +308,8 @@
                 base_module_name,))
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
+            i = lines.index('#include "_cffi_errors.h"\n')
+            lines[i:i+1] = self._rel_readlines('_cffi_errors.h')
             prnt(''.join(lines))
             self.needs_version(VERSION_EMBEDDED)
         #
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -174,10 +174,13 @@
 
     @jit.dont_look_inside
     def handle_applevel_exception(self, e, ll_res, extra_line):
+        from pypy.module._cffi_backend import errorbox
         space = self.space
         self.write_error_return_value(ll_res)
         if self.w_onerror is None:
+            ecap = errorbox.start_error_capture(space)
             self.print_error(e, extra_line)
+            errorbox.stop_error_capture(space, ecap)
         else:
             try:
                 e.normalize_exception(space)
@@ -189,10 +192,12 @@
                     self.convert_result(ll_res, w_res)
             except OperationError as e2:
                 # double exception! print a double-traceback...
+                ecap = errorbox.start_error_capture(space)
                 self.print_error(e, extra_line)    # original traceback
                 e2.write_unraisable(space, '', with_traceback=True,
                             extra_line="\nDuring the call to 'onerror', "
                                        "another exception occurred:\n\n")
+                errorbox.stop_error_capture(space, ecap)
 
 
 class W_CDataCallback(W_ExternPython):
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -63,6 +63,8 @@
             load_embedded_cffi_module(space, version, init_struct)
             res = 0
         except OperationError as operr:
+            from pypy.module._cffi_backend import errorbox
+            ecap = errorbox.start_error_capture(space)
             operr.write_unraisable(space, "initialization of '%s'" % name,
                                    with_traceback=True)
             space.appexec([], r"""():
@@ -71,6 +73,7 @@
                                  sys.pypy_version_info[:3])
                 sys.stderr.write('sys.path: %r\n' % (sys.path,))
             """)
+            errorbox.stop_error_capture(space, ecap)
             res = -1
         if must_leave:
             space.threadlocals.leave_thread(space)
diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -0,0 +1,114 @@
+# for Windows only
+import sys
+from rpython.rlib import jit
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+MESSAGEBOX = sys.platform == "win32"
+
+MODULE = r"""
+#include <Windows.h>
+#pragma comment(lib, "user32.lib")
+
+static void *volatile _cffi_bootstrap_text;
+
+RPY_EXTERN int _cffi_errorbox1(void)
+{
+    return InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+                                             (void *)1, NULL) == NULL;
+}
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "PyPy: Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+RPY_EXTERN void _cffi_errorbox(char *text)
+{
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+    HANDLE h;
+
+    _cffi_bootstrap_text = text;
+     h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                      NULL, 0, NULL);
+     if (h != NULL)
+         CloseHandle(h);
+}
+"""
+
+if MESSAGEBOX:
+
+    eci = ExternalCompilationInfo(
+            separate_module_sources=[MODULE],
+            post_include_bits=["RPY_EXTERN int _cffi_errorbox1(void);\n"
+                               "RPY_EXTERN void _cffi_errorbox(char *);\n"])
+
+    cffi_errorbox1 = rffi.llexternal("_cffi_errorbox1", [],
+                                     rffi.INT, compilation_info=eci)
+    cffi_errorbox = rffi.llexternal("_cffi_errorbox", [rffi.CCHARP],
+                                    lltype.Void, compilation_info=eci)
+
+    class Message:
+        def __init__(self, space):
+            self.space = space
+            self.text_p = lltype.nullptr(rffi.CCHARP.TO)
+
+        def start_error_capture(self):
+            ok = cffi_errorbox1()
+            if rffi.cast(lltype.Signed, ok) != 1:
+                return None
+
+            return self.space.appexec([], """():
+                import sys
+                class FileLike:
+                    def write(self, x):
+                        of.write(x)
+                        self.buf += x
+                fl = FileLike()
+                fl.buf = ''
+                of = sys.stderr
+                sys.stderr = fl
+                def done():
+                    sys.stderr = of
+                    return fl.buf
+                return done
+            """)
+
+        def stop_error_capture(self, w_done):
+            if w_done is None:
+                return
+
+            w_text = self.space.call_function(w_done)
+            # XXX Python 3: MessageBoxA() => MessageBoxW()
+
+            p = rffi.str2charp(self.space.bytes_w(w_text),
+                               track_allocation=False)
+            if self.text_p:
+                rffi.free_charp(self.text_p, track_allocation=False)
+            self.text_p = p      # keepalive
+
+            cffi_errorbox(p)
+
+
+    @jit.dont_look_inside
+    def start_error_capture(space):
+        msg = space.fromcache(Message)
+        return msg.start_error_capture()
+
+    @jit.dont_look_inside
+    def stop_error_capture(space, x):
+        msg = space.fromcache(Message)
+        msg.stop_error_capture(x)
+
+else:
+    def start_error_capture(space):
+        return None
+    def stop_error_capture(space, nothing):
+        pass
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
--- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
@@ -1,6 +1,10 @@
 /* Generated by pypy/tool/import_cffi.py */
 #include <stdio.h>
 
+#ifdef _MSC_VER
+#include <windows.h>
+#endif
+
 extern int add1(int, int);
 
 
@@ -10,5 +14,9 @@
     x = add1(40, 2);
     y = add1(100, -5);
     printf("got: %d %d\n", x, y);
+#ifdef _MSC_VER
+    if (x == 0 && y == 0)
+        Sleep(2000);
+#endif
     return 0;
 }
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
--- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
@@ -200,3 +200,9 @@
                           "prepADD2\n"
                           "adding 100 and -5 and -20\n"
                           "got: 42 75\n")
+
+    def test_init_time_error(self):
+        initerror_cffi = self.prepare_module('initerror')
+        self.compile('add1-test', [initerror_cffi])
+        output = self.execute('add1-test')
+        assert output == "got: 0 0\n"    # plus lots of info to stderr

From pypy.commits at gmail.com  Mon Jun 19 03:59:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 00:59:59 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5947847f.58abdf0a.11261.4d11@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91625:be1dadf96ba4
Date: 2017-06-19 09:59 +0200
http://bitbucket.org/pypy/pypy/changeset/be1dadf96ba4/

Log:	hg merge default

	also attempt to write the Python3 version of cffi's errorbox.py on
	Windows, but not tested so far

diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
new file mode 100644
--- /dev/null
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -0,0 +1,141 @@
+#ifndef CFFI_MESSAGEBOX
+# ifdef _MSC_VER
+#  define CFFI_MESSAGEBOX  1
+# else
+#  define CFFI_MESSAGEBOX  0
+# endif
+#endif
+
+
+#if CFFI_MESSAGEBOX
+/* Windows only: logic to take the Python-CFFI embedding logic
+   initialization errors and display them in a background thread
+   with MessageBox.  The idea is that if the whole program closes
+   as a result of this problem, then likely it is already a console
+   program and you can read the stderr output in the console too.
+   If it is not a console program, then it will likely show its own
+   dialog to complain, or generally not abruptly close, and for this
+   case the background thread should stay alive.
+*/
+static void *volatile _cffi_bootstrap_text;
+
+static PyObject *_cffi_start_error_capture(void)
+{
+    PyObject *result = NULL;
+    PyObject *x, *m, *bi;
+
+    if (InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+            (void *)1, NULL) != NULL)
+        return (PyObject *)1;
+
+    m = PyImport_AddModule("_cffi_error_capture");
+    if (m == NULL)
+        goto error;
+
+    result = PyModule_GetDict(m);
+    if (result == NULL)
+        goto error;
+
+    bi = PyImport_ImportModule("__builtin__");
+    if (bi == NULL)
+        goto error;
+    PyDict_SetItemString(result, "__builtins__", bi);
+    Py_DECREF(bi);
+
+    x = PyRun_String(
+        "import sys\n"
+        "class FileLike:\n"
+        "  def write(self, x):\n"
+        "    of.write(x)\n"
+        "    self.buf += x\n"
+        "fl = FileLike()\n"
+        "fl.buf = ''\n"
+        "of = sys.stderr\n"
+        "sys.stderr = fl\n"
+        "def done():\n"
+        "  sys.stderr = of\n"
+        "  return fl.buf\n",   /* make sure the returned value stays alive */
+        Py_file_input,
+        result, result);
+    Py_XDECREF(x);
+
+ error:
+    if (PyErr_Occurred())
+    {
+        PyErr_WriteUnraisable(Py_None);
+        PyErr_Clear();
+    }
+    return result;
+}
+
+#pragma comment(lib, "user32.lib")
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+#if PY_MAJOR_VERSION >= 3
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#else
+    MessageBoxA(NULL, (char *)_cffi_bootstrap_text,
+                "Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+#endif
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+static void _cffi_stop_error_capture(PyObject *ecap)
+{
+    PyObject *s;
+    void *text;
+
+    if (ecap == (PyObject *)1)
+        return;
+
+    if (ecap == NULL)
+        goto error;
+
+    s = PyRun_String("done()", Py_eval_input, ecap, ecap);
+    if (s == NULL)
+        goto error;
+
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+#if PY_MAJOR_VERSION >= 3
+    text = PyUnicode_AsWideCharString(s, NULL);
+#else
+    text = PyString_AsString(s);
+#endif
+
+    _cffi_bootstrap_text = text;
+
+    if (text != NULL)
+    {
+        HANDLE h;
+        h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                         NULL, 0, NULL);
+        if (h != NULL)
+            CloseHandle(h);
+    }
+    /* decref the string, but it should stay alive as 'fl.buf'
+       in the small module above.  It will really be freed only if
+       we later get another similar error.  So it's a leak of at
+       most one copy of the small module.  That's fine for this
+       situation which is usually a "fatal error" anyway. */
+    Py_DECREF(s);
+    PyErr_Clear();
+    return;
+
+  error:
+    _cffi_bootstrap_text = NULL;
+    PyErr_Clear();
+}
+
+#else
+
+static PyObject *_cffi_start_error_capture(void) { return NULL; }
+static void _cffi_stop_error_capture(PyObject *ecap) { }
+
+#endif
diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h
--- a/lib_pypy/cffi/_embedding.h
+++ b/lib_pypy/cffi/_embedding.h
@@ -109,6 +109,8 @@
 /**********  CPython-specific section  **********/
 #ifndef PYPY_VERSION
 
+#include "_cffi_errors.h"
+
 
 #define _cffi_call_python_org  _cffi_exports[_CFFI_CPIDX]
 
@@ -220,8 +222,16 @@
         /* Print as much information as potentially useful.
            Debugging load-time failures with embedding is not fun
         */
+        PyObject *ecap;
         PyObject *exception, *v, *tb, *f, *modules, *mod;
         PyErr_Fetch(&exception, &v, &tb);
+        ecap = _cffi_start_error_capture();
+        f = PySys_GetObject((char *)"stderr");
+        if (f != NULL && f != Py_None) {
+            PyFile_WriteString(
+                "Failed to initialize the Python-CFFI embedding logic:\n\n", f);
+        }
+
         if (exception != NULL) {
             PyErr_NormalizeException(&exception, &v, &tb);
             PyErr_Display(exception, v, tb);
@@ -230,7 +240,6 @@
         Py_XDECREF(v);
         Py_XDECREF(tb);
 
-        f = PySys_GetObject((char *)"stderr");
         if (f != NULL && f != Py_None) {
             PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
                                "\ncompiled with cffi version: 1.11.0"
@@ -249,6 +258,7 @@
             PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
             PyFile_WriteString("\n\n", f);
         }
+        _cffi_stop_error_capture(ecap);
     }
     result = -1;
     goto done;
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -75,9 +75,10 @@
         self._init_once_cache = {}
         self._cdef_version = None
         self._embedding = None
+        self._typecache = model.get_typecache(backend)
         if hasattr(backend, 'set_ffi'):
             backend.set_ffi(self)
-        for name in backend.__dict__:
+        for name in list(backend.__dict__):
             if name.startswith('RTLD_'):
                 setattr(self, name, getattr(backend, name))
         #
diff --git a/lib_pypy/cffi/model.py b/lib_pypy/cffi/model.py
--- a/lib_pypy/cffi/model.py
+++ b/lib_pypy/cffi/model.py
@@ -568,22 +568,26 @@
 
 
 global_lock = allocate_lock()
+_typecache_cffi_backend = weakref.WeakValueDictionary()
+
+def get_typecache(backend):
+    # returns _typecache_cffi_backend if backend is the _cffi_backend
+    # module, or type(backend).__typecache if backend is an instance of
+    # CTypesBackend (or some FakeBackend class during tests)
+    if isinstance(backend, types.ModuleType):
+        return _typecache_cffi_backend
+    with global_lock:
+        if not hasattr(type(backend), '__typecache'):
+            type(backend).__typecache = weakref.WeakValueDictionary()
+        return type(backend).__typecache
 
 def global_cache(srctype, ffi, funcname, *args, **kwds):
     key = kwds.pop('key', (funcname, args))
     assert not kwds
     try:
-        return ffi._backend.__typecache[key]
+        return ffi._typecache[key]
     except KeyError:
         pass
-    except AttributeError:
-        # initialize the __typecache attribute, either at the module level
-        # if ffi._backend is a module, or at the class level if ffi._backend
-        # is some instance.
-        if isinstance(ffi._backend, types.ModuleType):
-            ffi._backend.__typecache = weakref.WeakValueDictionary()
-        else:
-            type(ffi._backend).__typecache = weakref.WeakValueDictionary()
     try:
         res = getattr(ffi._backend, funcname)(*args)
     except NotImplementedError as e:
@@ -591,7 +595,7 @@
     # note that setdefault() on WeakValueDictionary is not atomic
     # and contains a rare bug (http://bugs.python.org/issue19542);
     # we have to use a lock and do it ourselves
-    cache = ffi._backend.__typecache
+    cache = ffi._typecache
     with global_lock:
         res1 = cache.get(key)
         if res1 is None:
diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py
--- a/lib_pypy/cffi/recompiler.py
+++ b/lib_pypy/cffi/recompiler.py
@@ -308,6 +308,8 @@
                 base_module_name,))
             prnt('#endif')
             lines = self._rel_readlines('_embedding.h')
+            i = lines.index('#include "_cffi_errors.h"\n')
+            lines[i:i+1] = self._rel_readlines('_cffi_errors.h')
             prnt(''.join(lines))
             self.needs_version(VERSION_EMBEDDED)
         #
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -174,10 +174,13 @@
 
     @jit.dont_look_inside
     def handle_applevel_exception(self, e, ll_res, extra_line):
+        from pypy.module._cffi_backend import errorbox
         space = self.space
         self.write_error_return_value(ll_res)
         if self.w_onerror is None:
+            ecap = errorbox.start_error_capture(space)
             self.print_error(e, extra_line)
+            errorbox.stop_error_capture(space, ecap)
         else:
             try:
                 e.normalize_exception(space)
@@ -189,10 +192,12 @@
                     self.convert_result(ll_res, w_res)
             except OperationError as e2:
                 # double exception! print a double-traceback...
+                ecap = errorbox.start_error_capture(space)
                 self.print_error(e, extra_line)    # original traceback
                 e2.write_unraisable(space, '', with_traceback=True,
                             extra_line="\nDuring the call to 'onerror', "
                                        "another exception occurred:\n\n")
+                errorbox.stop_error_capture(space, ecap)
 
 
 class W_CDataCallback(W_ExternPython):
diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py
--- a/pypy/module/_cffi_backend/embedding.py
+++ b/pypy/module/_cffi_backend/embedding.py
@@ -63,6 +63,8 @@
             load_embedded_cffi_module(space, version, init_struct)
             res = 0
         except OperationError as operr:
+            from pypy.module._cffi_backend import errorbox
+            ecap = errorbox.start_error_capture(space)
             operr.write_unraisable(space, "initialization of '%s'" % name,
                                    with_traceback=True)
             space.appexec([], r"""():
@@ -71,6 +73,7 @@
                                  sys.pypy_version_info[:3])
                 sys.stderr.write('sys.path: %r\n' % (sys.path,))
             """)
+            errorbox.stop_error_capture(space, ecap)
             res = -1
         if must_leave:
             space.threadlocals.leave_thread(space)
diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -0,0 +1,112 @@
+# for Windows only
+import sys
+from rpython.rlib import jit
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+MESSAGEBOX = sys.platform == "win32"
+
+MODULE = r"""
+#include <Windows.h>
+#pragma comment(lib, "user32.lib")
+
+static void *volatile _cffi_bootstrap_text;
+
+RPY_EXTERN int _cffi_errorbox1(void)
+{
+    return InterlockedCompareExchangePointer(&_cffi_bootstrap_text,
+                                             (void *)1, NULL) == NULL;
+}
+
+static DWORD WINAPI _cffi_bootstrap_dialog(LPVOID ignored)
+{
+    Sleep(666);    /* may be interrupted if the whole process is closing */
+    MessageBoxW(NULL, (wchar_t *)_cffi_bootstrap_text,
+                L"PyPy: Python-CFFI error",
+                MB_OK | MB_ICONERROR);
+    _cffi_bootstrap_text = NULL;
+    return 0;
+}
+
+RPY_EXTERN void _cffi_errorbox(wchar_t *text)
+{
+    /* Show a dialog box, but in a background thread, and
+       never show multiple dialog boxes at once. */
+    HANDLE h;
+
+    _cffi_bootstrap_text = text;
+     h = CreateThread(NULL, 0, _cffi_bootstrap_dialog,
+                      NULL, 0, NULL);
+     if (h != NULL)
+         CloseHandle(h);
+}
+"""
+
+if MESSAGEBOX:
+
+    eci = ExternalCompilationInfo(
+            separate_module_sources=[MODULE],
+            post_include_bits=["RPY_EXTERN int _cffi_errorbox1(void);\n"
+                               "RPY_EXTERN void _cffi_errorbox(wchar_t *);\n"])
+
+    cffi_errorbox1 = rffi.llexternal("_cffi_errorbox1", [],
+                                     rffi.INT, compilation_info=eci)
+    cffi_errorbox = rffi.llexternal("_cffi_errorbox", [rffi.CWCHARP],
+                                    lltype.Void, compilation_info=eci)
+
+    class Message:
+        def __init__(self, space):
+            self.space = space
+            self.text_p = lltype.nullptr(rffi.CWCHARP.TO)
+
+        def start_error_capture(self):
+            ok = cffi_errorbox1()
+            if rffi.cast(lltype.Signed, ok) != 1:
+                return None
+
+            return self.space.appexec([], """():
+                import sys
+                class FileLike:
+                    def write(self, x):
+                        of.write(x)
+                        self.buf += x
+                fl = FileLike()
+                fl.buf = ''
+                of = sys.stderr
+                sys.stderr = fl
+                def done():
+                    sys.stderr = of
+                    return fl.buf
+                return done
+            """)
+
+        def stop_error_capture(self, w_done):
+            if w_done is None:
+                return
+
+            w_text = self.space.call_function(w_done)
+            p = rffi.unicode2wcharp(self.space.text_w(w_text),
+                                    track_allocation=False)
+            if self.text_p:
+                rffi.free_wcharp(self.text_p, track_allocation=False)
+            self.text_p = p      # keepalive
+
+            cffi_errorbox(p)
+
+
+    @jit.dont_look_inside
+    def start_error_capture(space):
+        msg = space.fromcache(Message)
+        return msg.start_error_capture()
+
+    @jit.dont_look_inside
+    def stop_error_capture(space, x):
+        msg = space.fromcache(Message)
+        msg.stop_error_capture(x)
+
+else:
+    def start_error_capture(space):
+        return None
+    def stop_error_capture(space, nothing):
+        pass
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
--- a/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/add1-test.c
@@ -1,6 +1,10 @@
 /* Generated by pypy/tool/import_cffi.py */
 #include <stdio.h>
 
+#ifdef _MSC_VER
+#include <windows.h>
+#endif
+
 extern int add1(int, int);
 
 
@@ -10,5 +14,9 @@
     x = add1(40, 2);
     y = add1(100, -5);
     printf("got: %d %d\n", x, y);
+#ifdef _MSC_VER
+    if (x == 0 && y == 0)
+        Sleep(2000);
+#endif
     return 0;
 }
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
--- a/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/test_basic.py
@@ -200,3 +200,9 @@
                           "prepADD2\n"
                           "adding 100 and -5 and -20\n"
                           "got: 42 75\n")
+
+    def test_init_time_error(self):
+        initerror_cffi = self.prepare_module('initerror')
+        self.compile('add1-test', [initerror_cffi])
+        output = self.execute('add1-test')
+        assert output == "got: 0 0\n"    # plus lots of info to stderr

From pypy.commits at gmail.com  Mon Jun 19 04:12:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 01:12:59 -0700 (PDT)
Subject: [pypy-commit] cffi default: Py3 compat
Message-ID: <5947878b.423f1c0a.5d5a4.e156@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2982:7495a3729d42
Date: 2017-06-19 10:12 +0200
http://bitbucket.org/cffi/cffi/changeset/7495a3729d42/

Log:	Py3 compat

diff --git a/cffi/_cffi_errors.h b/cffi/_cffi_errors.h
--- a/cffi/_cffi_errors.h
+++ b/cffi/_cffi_errors.h
@@ -36,7 +36,11 @@
     if (result == NULL)
         goto error;
 
+#if PY_MAJOR_VERSION >= 3
+    bi = PyImport_ImportModule("builtins");
+#else
     bi = PyImport_ImportModule("__builtin__");
+#endif
     if (bi == NULL)
         goto error;
     PyDict_SetItemString(result, "__builtins__", bi);

From pypy.commits at gmail.com  Mon Jun 19 04:56:15 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 01:56:15 -0700 (PDT)
Subject: [pypy-commit] cffi default: Some test fixes for Python 3 on Windows
Message-ID: <594791af.86941c0a.228ef.a1ff@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2983:09cc68d4b9cf
Date: 2017-06-19 10:55 +0200
http://bitbucket.org/cffi/cffi/changeset/09cc68d4b9cf/

Log:	Some test fixes for Python 3 on Windows

diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -61,6 +61,10 @@
         path = None
     else:
         path = ctypes.util.find_library(name)
+        if path is None and name == 'c':
+            assert sys.platform == 'win32'
+            assert sys.version_info >= (3,)
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     return load_library(path, flags)
 
 def test_load_library():
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -765,7 +765,7 @@
         if sys.platform != "win32":
             return backend.load_library(None, flags)
         name = "c"    # Windows: load_library(None) fails, but this works
-                      # (backward compatibility hack only)
+                      # on Python 2 (backward compatibility hack only)
     first_error = None
     if '.' in name or '/' in name or os.sep in name:
         try:
@@ -775,6 +775,9 @@
     import ctypes.util
     path = ctypes.util.find_library(name)
     if path is None:
+        if name == "c" and sys.platform == "win32" and sys.version_info >= (3,):
+            raise OSError("dlopen(None) cannot work on Windows for Python 3 "
+                          "(see http://bugs.python.org/issue23606)")
         msg = ("ctypes.util.find_library() did not manage "
                "to locate a library called %r" % (name,))
         if first_error is not None:
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -306,7 +306,10 @@
 contain a full path or not (in which case it is searched in standard
 locations, as described in ``man dlopen``), with extensions or not.
 Alternatively, if ``libpath`` is None, it returns the standard C library
-(which can be used to access the functions of glibc, on Linux).
+(which can be used to access the functions of glibc, on Linux).  Note
+that ``libpath`` `cannot be None`__ on Windows with Python 3.
+
+.. __: http://bugs.python.org/issue23606
 
 Let me state it again: this gives ABI-level access to the library, so
 you need to have all types declared manually exactly as they were
diff --git a/testing/cffi0/backend_tests.py b/testing/cffi0/backend_tests.py
--- a/testing/cffi0/backend_tests.py
+++ b/testing/cffi0/backend_tests.py
@@ -10,6 +10,10 @@
 SIZE_OF_PTR   = ctypes.sizeof(ctypes.c_void_p)
 SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar)
 
+def needs_dlopen_none():
+    if sys.platform == 'win32' and sys.version_info >= (3,):
+        py.test.skip("dlopen(None) cannot work on Windows for Python 3")
+
 
 class BackendTests:
 
@@ -354,7 +358,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
@@ -937,6 +940,7 @@
     def test_enum_partial(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef(r"enum foo {A, ...}; enum bar { B, C };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.B == 0
         py.test.raises(VerificationMissing, getattr, lib, "A")
@@ -1844,6 +1848,7 @@
             #define DOT_UL 1000UL
             enum foo {AA, BB=DOT, CC};
         """)
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert ffi.string(ffi.cast("enum foo", 100)) == "BB"
         assert lib.DOT_0 == 0
@@ -1873,6 +1878,7 @@
         ffi = FFI()
         ffi.include(ffi1)
         ffi.cdef("enum { EE2, EE3 };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.EE1 == 0
         assert lib.EE2 == 0
diff --git a/testing/cffi0/test_function.py b/testing/cffi0/test_function.py
--- a/testing/cffi0/test_function.py
+++ b/testing/cffi0/test_function.py
@@ -5,6 +5,7 @@
 from cffi.backend_ctypes import CTypesBackend
 from testing.udir import udir
 from testing.support import FdWriteCapture
+from .backend_tests import needs_dlopen_none
 
 try:
     from StringIO import StringIO
@@ -111,6 +112,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -127,6 +129,7 @@
             int fputs(char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -143,6 +146,7 @@
            int fprintf(void *, const char *format, ...);
            void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         with FdWriteCapture() as fd:
             ffi.C.fprintf(ffi.C.stderr, b"hello with no arguments\n")
@@ -169,6 +173,7 @@
         ffi.cdef("""
            int printf(const char *format, ...);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         e = py.test.raises(TypeError, ffi.C.printf, b"hello %d\n", 42)
         assert str(e.value) == ("argument 2 passed in the variadic part "
@@ -179,6 +184,7 @@
         ffi.cdef("""
             int puts(const char *);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.C.puts
         assert ffi.typeof(fptr) == ffi.typeof("int(*)(const char*)")
@@ -202,6 +208,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.cast("int(*)(const char *txt, void *)", ffi.C.fputs)
         assert fptr == ffi.C.fputs
@@ -235,6 +242,7 @@
         ffi.cdef("""
             int strlen(char[]);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello")
         res = ffi.C.strlen(p)
@@ -247,6 +255,7 @@
         ffi.cdef("""
             void *stdout;
         """)
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         pout = C.stdout
         C.stdout = ffi.NULL
@@ -259,6 +268,7 @@
         ffi.cdef("""
             char *strchr(const char *s, int c);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello world!")
         q = ffi.C.strchr(p, ord('w'))
@@ -275,6 +285,7 @@
             struct in_addr { unsigned int s_addr; };
             char *inet_ntoa(struct in_addr in);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ina = ffi.new("struct in_addr *", [0x04040404])
         a = ffi.C.inet_ntoa(ina[0])
@@ -296,6 +307,7 @@
         filename = str(udir.join('fputs_custom_FILE'))
         ffi = FFI(backend=self.Backend())
         ffi.cdef("int fputs(const char *, FILE *);")
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         with open(filename, 'wb') as f:
             f.write(b'[')
@@ -311,6 +323,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""enum foo_e { AA, BB, CC=5, DD };
                     typedef enum { EE=-5, FF } some_enum_t;""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.AA == 0
         assert lib.BB == 1
@@ -322,6 +335,7 @@
     def test_void_star_accepts_string(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(const void *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -331,6 +345,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(signed char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -340,6 +355,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(unsigned char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
diff --git a/testing/cffi0/test_parsing.py b/testing/cffi0/test_parsing.py
--- a/testing/cffi0/test_parsing.py
+++ b/testing/cffi0/test_parsing.py
@@ -1,5 +1,7 @@
 import py, sys, re
 from cffi import FFI, FFIError, CDefError, VerificationError
+from .backend_tests import needs_dlopen_none
+
 
 class FakeBackend(object):
 
@@ -90,6 +92,7 @@
 def test_pipe():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("int pipe(int pipefd[2]);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.pipe
     assert func.name == 'pipe'
@@ -98,6 +101,7 @@
 def test_vararg():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("short foo(int, ...);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.foo
     assert func.name == 'foo'
@@ -108,6 +112,7 @@
     ffi.cdef("""
         int foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.foo.BType == '<func (), <int>, False>'
 
@@ -118,6 +123,7 @@
         typedef UInt UIntReally;
         UInt foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("UIntReally")) == '<unsigned int>'
     assert C.foo.BType == '<func (), <unsigned int>, False>'
@@ -128,6 +134,7 @@
         typedef struct { int a, b; } foo_t, *foo_p;
         int foo(foo_p[]);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("foo_t")) == '<int>a, <int>b'
     assert str(ffi.typeof("foo_p")) == '<pointer to <int>a, <int>b>'
@@ -217,6 +224,7 @@
 
 def test_override():
     ffi = FFI(backend=FakeBackend())
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     ffi.cdef("int foo(void);")
     py.test.raises(FFIError, ffi.cdef, "long foo(void);")
@@ -403,6 +411,7 @@
     ffi.cdef("""
         enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.POS == 1
     assert C.TWO == 2

From pypy.commits at gmail.com  Mon Jun 19 06:09:57 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 03:09:57 -0700 (PDT)
Subject: [pypy-commit] cffi default: Don't use error capture around ffi.gc()
 finalizer errors
Message-ID: <5947a2f5.898adf0a.afdbc.c763@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2984:1d3da4b31373
Date: 2017-06-19 11:58 +0200
http://bitbucket.org/cffi/cffi/changeset/1d3da4b31373/

Log:	Don't use error capture around ffi.gc() finalizer errors

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -1852,12 +1852,15 @@
             Py_DECREF(result);
         }
         else {
-            PyObject *ecap, *t, *v, *tb;
+            PyObject *t, *v, *tb;
             PyErr_Fetch(&t, &v, &tb);
-            ecap = _cffi_start_error_capture();
+            /* Don't use error capture here, because it is very much
+             * like errors at __del__(), and these ones are not captured
+             * either */
+            /* ecap = _cffi_start_error_capture(); */
             _my_PyErr_WriteUnraisable(t, v, tb, "From callback for ffi.gc ",
                                       origobj, NULL);
-            _cffi_stop_error_capture(ecap);
+            /* _cffi_stop_error_capture(ecap); */
         }
         Py_DECREF(destructor);
 

From pypy.commits at gmail.com  Mon Jun 19 06:09:59 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 03:09:59 -0700 (PDT)
Subject: [pypy-commit] cffi default: Write a copy of wchar_helper.h that
 uses CPython 3.3's new
Message-ID: <5947a2f7.81851c0a.7e657.7b8e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2985:5e554e5c0dfc
Date: 2017-06-19 12:06 +0200
http://bitbucket.org/cffi/cffi/changeset/5e554e5c0dfc/

Log:	Write a copy of wchar_helper.h that uses CPython 3.3's new unicode
	string API. It makes sense anyway for speed reasons, but it also
	avoids even more special cases for platforms with 16-bit wchar_t on
	CPython >= 3.3.

diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -286,7 +286,11 @@
 # include "file_emulator.h"
 #endif
 
-#include "wchar_helper.h"
+#ifdef PyUnicode_KIND     /* Python >= 3.3 */
+# include "wchar_helper_3.h"
+#else
+# include "wchar_helper.h"
+#endif
 
 #include "../cffi/_cffi_errors.h"
 
diff --git a/c/wchar_helper_3.h b/c/wchar_helper_3.h
new file mode 100644
--- /dev/null
+++ b/c/wchar_helper_3.h
@@ -0,0 +1,155 @@
+/*
+ * wchar_t helpers, version CPython >= 3.3.
+ *
+ * CPython 3.3 added support for sys.maxunicode == 0x10FFFF on all
+ * platforms, even ones with wchar_t limited to 2 bytes.  As such,
+ * this code here works from the outside like wchar_helper.h in the
+ * case Py_UNICODE_SIZE == 4, but the implementation is very different.
+ */
+
+typedef uint16_t cffi_char16_t;
+typedef uint32_t cffi_char32_t;
+
+
+static PyObject *
+_my_PyUnicode_FromChar32(const cffi_char32_t *w, Py_ssize_t size)
+{
+    return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, w, size);
+}
+
+static PyObject *
+_my_PyUnicode_FromChar16(const cffi_char16_t *w, Py_ssize_t size)
+{
+    /* are there any surrogate pairs, and if so, how many? */
+    Py_ssize_t i, count_surrogates = 0;
+    for (i = 0; i < size - 1; i++) {
+        if (0xD800 <= w[i] && w[i] <= 0xDBFF &&
+                0xDC00 <= w[i+1] && w[i+1] <= 0xDFFF)
+            count_surrogates++;
+    }
+    if (count_surrogates == 0) {
+        /* no, fast path */
+        return PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, w, size);
+    }
+    else
+    {
+        PyObject *result = PyUnicode_New(size - count_surrogates, 0x10FFFF);
+        Py_UCS4 *data;
+        assert(PyUnicode_KIND(result) == PyUnicode_4BYTE_KIND);
+        data = PyUnicode_4BYTE_DATA(result);
+
+        for (i = 0; i < size; i++)
+        {
+            cffi_char32_t ch = w[i];
+            if (0xD800 <= ch && ch <= 0xDBFF && i < size - 1) {
+                cffi_char32_t ch2 = w[i + 1];
+                if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
+                    ch = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
+                    i++;
+                }
+            }
+            *data++ = ch;
+        }
+        return result;
+    }
+}
+
+static int
+_my_PyUnicode_AsSingleChar16(PyObject *unicode, cffi_char16_t *result,
+                             char *err_got)
+{
+    cffi_char32_t ch;
+    if (PyUnicode_GET_LENGTH(unicode) != 1) {
+        sprintf(err_got, "unicode string of length %zd",
+                PyUnicode_GET_LENGTH(unicode));
+        return -1;
+    }
+    ch = PyUnicode_READ_CHAR(unicode, 0);
+
+    if (ch > 0xFFFF)
+    {
+        sprintf(err_got, "larger-than-0xFFFF character");
+        return -1;
+    }
+    *result = (cffi_char16_t)ch;
+    return 0;
+}
+
+static int
+_my_PyUnicode_AsSingleChar32(PyObject *unicode, cffi_char32_t *result,
+                             char *err_got)
+{
+    if (PyUnicode_GET_LENGTH(unicode) != 1) {
+        sprintf(err_got, "unicode string of length %zd",
+                PyUnicode_GET_LENGTH(unicode));
+        return -1;
+    }
+    *result = PyUnicode_READ_CHAR(unicode, 0);
+    return 0;
+}
+
+static Py_ssize_t _my_PyUnicode_SizeAsChar16(PyObject *unicode)
+{
+    Py_ssize_t length = PyUnicode_GET_LENGTH(unicode);
+    Py_ssize_t result = length;
+    unsigned int kind = PyUnicode_KIND(unicode);
+
+    if (kind == PyUnicode_4BYTE_KIND)
+    {
+        Py_UCS4 *data = PyUnicode_4BYTE_DATA(unicode);
+        Py_ssize_t i;
+        for (i = 0; i < length; i++) {
+            if (data[i] > 0xFFFF)
+                result++;
+        }
+    }
+    return result;
+}
+
+static Py_ssize_t _my_PyUnicode_SizeAsChar32(PyObject *unicode)
+{
+    return PyUnicode_GET_LENGTH(unicode);
+}
+
+static int _my_PyUnicode_AsChar16(PyObject *unicode,
+                                  cffi_char16_t *result,
+                                  Py_ssize_t resultlen)
+{
+    Py_ssize_t len = PyUnicode_GET_LENGTH(unicode);
+    unsigned int kind = PyUnicode_KIND(unicode);
+    void *data = PyUnicode_DATA(unicode);
+    Py_ssize_t i;
+
+    for (i = 0; i < len; i++) {
+        cffi_char32_t ordinal = PyUnicode_READ(kind, data, i);
+        if (ordinal > 0xFFFF) {
+            if (ordinal > 0x10FFFF) {
+                PyErr_Format(PyExc_ValueError,
+                             "unicode character out of range for "
+                             "conversion to char16_t: 0x%x", (int)ordinal);
+                return -1;
+            }
+            ordinal -= 0x10000;
+            *result++ = 0xD800 | (ordinal >> 10);
+            *result++ = 0xDC00 | (ordinal & 0x3FF);
+        }
+        else
+            *result++ = ordinal;
+    }
+    return 0;
+}
+
+static int _my_PyUnicode_AsChar32(PyObject *unicode,
+                                  cffi_char32_t *result,
+                                  Py_ssize_t resultlen)
+{
+    Py_ssize_t len = PyUnicode_GET_LENGTH(unicode);
+    unsigned int kind = PyUnicode_KIND(unicode);
+    void *data = PyUnicode_DATA(unicode);
+    Py_ssize_t i;
+
+    for (i = 0; i < len; i++)
+        result[i] = PyUnicode_READ(kind, data, i);
+
+    return 0;
+}

From pypy.commits at gmail.com  Mon Jun 19 06:35:29 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 03:35:29 -0700 (PDT)
Subject: [pypy-commit] cffi default: More test fixes on Windows on Python 3
Message-ID: <5947a8f1.c9321c0a.c2e89.2594@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2987:b4e78ef300b0
Date: 2017-06-19 12:27 +0200
http://bitbucket.org/cffi/cffi/changeset/b4e78ef300b0/

Log:	More test fixes on Windows on Python 3

diff --git a/testing/cffi1/test_new_ffi_1.py b/testing/cffi1/test_new_ffi_1.py
--- a/testing/cffi1/test_new_ffi_1.py
+++ b/testing/cffi1/test_new_ffi_1.py
@@ -418,7 +418,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
diff --git a/testing/cffi1/test_re_python.py b/testing/cffi1/test_re_python.py
--- a/testing/cffi1/test_re_python.py
+++ b/testing/cffi1/test_re_python.py
@@ -94,6 +94,8 @@
     if sys.platform == 'win32':
         import ctypes.util
         name = ctypes.util.find_msvcrt()
+        if name is None:
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     lib = ffi.dlopen(name)
     assert lib.strlen(b"hello") == 5
 

From pypy.commits at gmail.com  Mon Jun 19 06:35:31 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 03:35:31 -0700 (PDT)
Subject: [pypy-commit] cffi default: bah,
 I won't attempt to find the correct #ifdef magic to use on MSVC in
 C++ mode to know if 'char16_t' exists. The official one doesn't work,
 so meh.
Message-ID: <5947a8f3.98b1df0a.1158e.b1ec@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2988:40a258cefe22
Date: 2017-06-19 12:35 +0200
http://bitbucket.org/cffi/cffi/changeset/40a258cefe22/

Log:	bah, I won't attempt to find the correct #ifdef magic to use on MSVC
	in C++ mode to know if 'char16_t' exists. The official one doesn't
	work, so meh.

diff --git a/testing/cffi1/test_recompiler.py b/testing/cffi1/test_recompiler.py
--- a/testing/cffi1/test_recompiler.py
+++ b/testing/cffi1/test_recompiler.py
@@ -2259,6 +2259,9 @@
     assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
 
 def test_char16_char32_type(no_cpp=False):
+    if no_cpp is False and sys.platform == "win32":
+        py.test.skip("aaaaaaa why do modern MSVC compilers still define "
+                     "a very old __cplusplus value")
     ffi = FFI()
     ffi.cdef("""
         char16_t foo_2bytes(char16_t);
@@ -2269,7 +2272,7 @@
     typedef uint_least16_t char16_t;
     typedef uint_least32_t char32_t;
     #endif
-    
+
     char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
     char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
     """, no_cpp=no_cpp)

From pypy.commits at gmail.com  Mon Jun 19 06:35:27 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 19 Jun 2017 03:35:27 -0700 (PDT)
Subject: [pypy-commit] cffi default: Probable improvement
Message-ID: <5947a8ef.6c8fdf0a.de85d.910d@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2986:54a7350ed586
Date: 2017-06-19 12:26 +0200
http://bitbucket.org/cffi/cffi/changeset/54a7350ed586/

Log:	Probable improvement

diff --git a/c/wchar_helper_3.h b/c/wchar_helper_3.h
--- a/c/wchar_helper_3.h
+++ b/c/wchar_helper_3.h
@@ -143,13 +143,7 @@
                                   cffi_char32_t *result,
                                   Py_ssize_t resultlen)
 {
-    Py_ssize_t len = PyUnicode_GET_LENGTH(unicode);
-    unsigned int kind = PyUnicode_KIND(unicode);
-    void *data = PyUnicode_DATA(unicode);
-    Py_ssize_t i;
-
-    for (i = 0; i < len; i++)
-        result[i] = PyUnicode_READ(kind, data, i);
-
+    if (PyUnicode_AsUCS4(unicode, (Py_UCS4 *)result, resultlen, 0) == NULL)
+        return -1;
     return 0;
 }

From pypy.commits at gmail.com  Mon Jun 19 13:55:37 2017
From: pypy.commits at gmail.com (mattip)
Date: Mon, 19 Jun 2017 10:55:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: make FrozenDict fail PySequence_Check()
 - fixes obscure numpy test failure
Message-ID: <59481019.4d8edf0a.e0032.0703@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91626:6d81023ea3fe
Date: 2017-06-19 20:01 +0300
http://bitbucket.org/pypy/pypy/changeset/6d81023ea3fe/

Log:	make FrozenDict fail PySequence_Check() - fixes obscure numpy test
	failure

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -290,6 +290,7 @@
 def make_frozendict(space):
     if space not in _frozendict_cache:
         _frozendict_cache[space] = _make_frozendict(space)
+        _frozendict_cache[space].flag_map_or_seq = 'M'
     return _frozendict_cache[space]
 
 _frozendict_cache = {}
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -33,6 +33,11 @@
                 return obj;
             '''
             ),
+           ("get_dictproxy", "METH_O",
+            '''
+                return PyDictProxy_New(args);
+            '''
+            ),
            ("check", "METH_O",
             '''
                 return PyInt_FromLong(
@@ -74,6 +79,10 @@
         assert operator.isMappingType(obj)
         #
         assert module.check(obj) == 2
+        # make sure dictionaries return false for PySequence_Check
+        assert module.check({'a': 1}) == 2
+        obj = module.get_dictproxy({'a': 10})
+        assert module.check(obj) == 2
 
     def test_iterable_nonmapping_object(self):
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Mon Jun 19 14:38:52 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 19 Jun 2017 11:38:52 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <59481a3c.d399df0a.ea9bc.5ace@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91628:44469c25a324
Date: 2017-06-19 19:21 +0100
http://bitbucket.org/pypy/pypy/changeset/44469c25a324/

Log:	fix test

diff --git a/pypy/module/imp/test/test_app.py b/pypy/module/imp/test/test_app.py
--- a/pypy/module/imp/test/test_app.py
+++ b/pypy/module/imp/test/test_app.py
@@ -115,6 +115,7 @@
             mod = imp.init_builtin(name)
             assert mod
             assert mod.__spec__
+    test_is_builtin.dont_track_allocations = True
 
     def test_load_module_py(self):
         import imp

From pypy.commits at gmail.com  Mon Jun 19 14:38:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 19 Jun 2017 11:38:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test_mixedmodule.py
Message-ID: <59481a3a.103a1c0a.83826.cc09@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91627:675b2f4d2399
Date: 2017-06-19 17:41 +0100
http://bitbucket.org/pypy/pypy/changeset/675b2f4d2399/

Log:	fix test_mixedmodule.py

diff --git a/pypy/interpreter/test/test_mixedmodule.py b/pypy/interpreter/test/test_mixedmodule.py
--- a/pypy/interpreter/test/test_mixedmodule.py
+++ b/pypy/interpreter/test/test_mixedmodule.py
@@ -1,38 +1,43 @@
+import pytest
+
+from pypy.tool.pytest.objspace import maketestobjspace
 from pypy.interpreter.mixedmodule import MixedModule
-import py.test
 
+ at pytest.fixture()
+def space():
+    # We need a fresh space for each test here
+    return maketestobjspace()
 
-class TestMixedModule(object):
-    def test_install(self):
-        class Module(MixedModule):
-            interpleveldefs = {}
-            appleveldefs = {}
+def test_install(space):
+    class Module(MixedModule):
+        interpleveldefs = {}
+        appleveldefs = {}
 
-        m = Module(self.space, self.space.wrap("test_module"))
-        m.install()
+    m = Module(space, space.wrap("test_module"))
+    m.install()
 
-        assert self.space.builtin_modules["test_module"] is m
+    assert space.builtin_modules["test_module"] is m
 
-    def test_submodule(self):
-        class SubModule(MixedModule):
-            interpleveldefs = {}
-            appleveldefs = {}
+def test_submodule(space):
+    class SubModule(MixedModule):
+        interpleveldefs = {}
+        appleveldefs = {}
 
-        class Module(MixedModule):
-            interpleveldefs = {}
-            appleveldefs = {}
-            submodules = {
-                "sub": SubModule
-            }
+    class Module(MixedModule):
+        interpleveldefs = {}
+        appleveldefs = {}
+        submodules = {
+            "sub": SubModule
+        }
 
-        m = Module(self.space, self.space.wrap("test_module"))
-        m.install()
+    m = Module(space, space.wrap("test_module"))
+    m.install()
 
-        assert self.space.builtin_modules["test_module"] is m
-        assert isinstance(self.space.builtin_modules["test_module.sub"], SubModule)
+    assert space.builtin_modules["test_module"] is m
+    assert isinstance(space.builtin_modules["test_module.sub"], SubModule)
 
 class AppTestMixedModule(object):
-    pytestmark = py.test.mark.skipif("config.option.runappdirect")
+    pytestmark = pytest.mark.skipif("config.option.runappdirect")
 
     def setup_class(cls):
         space = cls.space

From pypy.commits at gmail.com  Tue Jun 20 07:09:12 2017
From: pypy.commits at gmail.com (arigo)
Date: Tue, 20 Jun 2017 04:09:12 -0700 (PDT)
Subject: [pypy-commit] cffi default: List _cffi_errors.h here (oops,
 thanks gabriel-m)
Message-ID: <59490258.08aadf0a.2e2ed.8785@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2989:f06010593f74
Date: 2017-06-20 13:08 +0200
http://bitbucket.org/cffi/cffi/changeset/f06010593f74/

Log:	List _cffi_errors.h here (oops, thanks gabriel-m)

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -186,7 +186,7 @@
         version='1.11.0',
         packages=['cffi'] if cpython else [],
         package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h', 
-                               '_embedding.h']}
+                               '_embedding.h', '_cffi_errors.h']}
                      if cpython else {},
         zip_safe=False,
 

From pypy.commits at gmail.com  Wed Jun 21 05:42:06 2017
From: pypy.commits at gmail.com (arigo)
Date: Wed, 21 Jun 2017 02:42:06 -0700 (PDT)
Subject: [pypy-commit] pypy default: update to cffi/f06010593f74
Message-ID: <594a3f6e.c385df0a.fbb56.fc0e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91629:4b2c32aaaf36
Date: 2017-06-21 11:40 +0200
http://bitbucket.org/pypy/pypy/changeset/4b2c32aaaf36/

Log:	update to cffi/f06010593f74

diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -36,7 +36,11 @@
     if (result == NULL)
         goto error;
 
+#if PY_MAJOR_VERSION >= 3
+    bi = PyImport_ImportModule("builtins");
+#else
     bi = PyImport_ImportModule("__builtin__");
+#endif
     if (bi == NULL)
         goto error;
     PyDict_SetItemString(result, "__builtins__", bi);
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -765,7 +765,7 @@
         if sys.platform != "win32":
             return backend.load_library(None, flags)
         name = "c"    # Windows: load_library(None) fails, but this works
-                      # (backward compatibility hack only)
+                      # on Python 2 (backward compatibility hack only)
     first_error = None
     if '.' in name or '/' in name or os.sep in name:
         try:
@@ -775,6 +775,9 @@
     import ctypes.util
     path = ctypes.util.find_library(name)
     if path is None:
+        if name == "c" and sys.platform == "win32" and sys.version_info >= (3,):
+            raise OSError("dlopen(None) cannot work on Windows for Python 3 "
+                          "(see http://bugs.python.org/issue23606)")
         msg = ("ctypes.util.find_library() did not manage "
                "to locate a library called %r" % (name,))
         if first_error is not None:
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -50,6 +50,10 @@
         path = None
     else:
         path = ctypes.util.find_library(name)
+        if path is None and name == 'c':
+            assert sys.platform == 'win32'
+            assert sys.version_info >= (3,)
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     return load_library(path, flags)
 
 def test_load_library():
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -11,6 +11,10 @@
 SIZE_OF_PTR   = ctypes.sizeof(ctypes.c_void_p)
 SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar)
 
+def needs_dlopen_none():
+    if sys.platform == 'win32' and sys.version_info >= (3,):
+        py.test.skip("dlopen(None) cannot work on Windows for Python 3")
+
 
 class BackendTests:
 
@@ -355,7 +359,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
@@ -938,6 +941,7 @@
     def test_enum_partial(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef(r"enum foo {A, ...}; enum bar { B, C };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.B == 0
         py.test.raises(VerificationMissing, getattr, lib, "A")
@@ -1845,6 +1849,7 @@
             #define DOT_UL 1000UL
             enum foo {AA, BB=DOT, CC};
         """)
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert ffi.string(ffi.cast("enum foo", 100)) == "BB"
         assert lib.DOT_0 == 0
@@ -1874,6 +1879,7 @@
         ffi = FFI()
         ffi.include(ffi1)
         ffi.cdef("enum { EE2, EE3 };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.EE1 == 0
         assert lib.EE2 == 0
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
@@ -6,6 +6,7 @@
 from cffi.backend_ctypes import CTypesBackend
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture
+from .backend_tests import needs_dlopen_none
 
 try:
     from StringIO import StringIO
@@ -112,6 +113,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -128,6 +130,7 @@
             int fputs(char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -144,6 +147,7 @@
            int fprintf(void *, const char *format, ...);
            void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         with FdWriteCapture() as fd:
             ffi.C.fprintf(ffi.C.stderr, b"hello with no arguments\n")
@@ -170,6 +174,7 @@
         ffi.cdef("""
            int printf(const char *format, ...);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         e = py.test.raises(TypeError, ffi.C.printf, b"hello %d\n", 42)
         assert str(e.value) == ("argument 2 passed in the variadic part "
@@ -180,6 +185,7 @@
         ffi.cdef("""
             int puts(const char *);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.C.puts
         assert ffi.typeof(fptr) == ffi.typeof("int(*)(const char*)")
@@ -203,6 +209,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.cast("int(*)(const char *txt, void *)", ffi.C.fputs)
         assert fptr == ffi.C.fputs
@@ -236,6 +243,7 @@
         ffi.cdef("""
             int strlen(char[]);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello")
         res = ffi.C.strlen(p)
@@ -248,6 +256,7 @@
         ffi.cdef("""
             void *stdout;
         """)
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         pout = C.stdout
         C.stdout = ffi.NULL
@@ -260,6 +269,7 @@
         ffi.cdef("""
             char *strchr(const char *s, int c);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello world!")
         q = ffi.C.strchr(p, ord('w'))
@@ -276,6 +286,7 @@
             struct in_addr { unsigned int s_addr; };
             char *inet_ntoa(struct in_addr in);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ina = ffi.new("struct in_addr *", [0x04040404])
         a = ffi.C.inet_ntoa(ina[0])
@@ -297,6 +308,7 @@
         filename = str(udir.join('fputs_custom_FILE'))
         ffi = FFI(backend=self.Backend())
         ffi.cdef("int fputs(const char *, FILE *);")
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         with open(filename, 'wb') as f:
             f.write(b'[')
@@ -312,6 +324,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""enum foo_e { AA, BB, CC=5, DD };
                     typedef enum { EE=-5, FF } some_enum_t;""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.AA == 0
         assert lib.BB == 1
@@ -323,6 +336,7 @@
     def test_void_star_accepts_string(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(const void *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -332,6 +346,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(signed char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -341,6 +356,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(unsigned char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -1,6 +1,8 @@
 # Generated by pypy/tool/import_cffi.py
 import py, sys, re
 from cffi import FFI, FFIError, CDefError, VerificationError
+from .backend_tests import needs_dlopen_none
+
 
 class FakeBackend(object):
 
@@ -91,6 +93,7 @@
 def test_pipe():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("int pipe(int pipefd[2]);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.pipe
     assert func.name == 'pipe'
@@ -99,6 +102,7 @@
 def test_vararg():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("short foo(int, ...);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.foo
     assert func.name == 'foo'
@@ -109,6 +113,7 @@
     ffi.cdef("""
         int foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.foo.BType == '<func (), <int>, False>'
 
@@ -119,6 +124,7 @@
         typedef UInt UIntReally;
         UInt foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("UIntReally")) == '<unsigned int>'
     assert C.foo.BType == '<func (), <unsigned int>, False>'
@@ -129,6 +135,7 @@
         typedef struct { int a, b; } foo_t, *foo_p;
         int foo(foo_p[]);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("foo_t")) == '<int>a, <int>b'
     assert str(ffi.typeof("foo_p")) == '<pointer to <int>a, <int>b>'
@@ -218,6 +225,7 @@
 
 def test_override():
     ffi = FFI(backend=FakeBackend())
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     ffi.cdef("int foo(void);")
     py.test.raises(FFIError, ffi.cdef, "long foo(void);")
@@ -404,6 +412,7 @@
     ffi.cdef("""
         enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.POS == 1
     assert C.TWO == 2
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -419,7 +419,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
@@ -95,6 +95,8 @@
     if sys.platform == 'win32':
         import ctypes.util
         name = ctypes.util.find_msvcrt()
+        if name is None:
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     lib = ffi.dlopen(name)
     assert lib.strlen(b"hello") == 5
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2260,6 +2260,9 @@
     assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
 
 def test_char16_char32_type(no_cpp=False):
+    if no_cpp is False and sys.platform == "win32":
+        py.test.skip("aaaaaaa why do modern MSVC compilers still define "
+                     "a very old __cplusplus value")
     ffi = FFI()
     ffi.cdef("""
         char16_t foo_2bytes(char16_t);
@@ -2270,7 +2273,7 @@
     typedef uint_least16_t char16_t;
     typedef uint_least32_t char32_t;
     #endif
-    
+
     char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
     char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
     """, no_cpp=no_cpp)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py
@@ -0,0 +1,19 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+    int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+    raise KeyError
+""")
+
+ffi.set_source("_initerror_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
+

From pypy.commits at gmail.com  Wed Jun 21 17:09:37 2017
From: pypy.commits at gmail.com (mattip)
Date: Wed, 21 Jun 2017 14:09:37 -0700 (PDT)
Subject: [pypy-commit] pypy default: update help-needed section
Message-ID: <594ae091.c385df0a.fbb56.c506@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91630:471848fce43a
Date: 2017-06-22 00:08 +0300
http://bitbucket.org/pypy/pypy/changeset/471848fce43a/

Log:	update help-needed section

diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -238,18 +238,17 @@
 using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
 good work that needs to be finished:
 
-**matplotlib** https://github.com/mattip/matplotlib
+**matplotlib** https://github.com/matplotlib/matplotlib
 
-    Status: the repo is an older version of matplotlib adapted to pypy and cpyext
+    TODO: the tkagg backend does not work, which makes tests fail on downstream
+    projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in 
+    tkagg.py, which requires refactoring
 
-    TODO: A suggested first step would be to merge the differences into 
-    matplotlib/HEAD. The major problem is the use of a generic view into a
-    numpy ndarray. The int* fields would need to be converted into int[MAX_DIMS]
-    c-arrays and filled in.
+**wxPython** https://bitbucket.org/amauryfa/wxpython-cffi
 
-**wxPython** https://bitbucket.org/waedt/wxpython_cffi
+    Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi
 
-    Status: A GSOC 2013 project to adapt the Phoenix sip build system to cffi
+    The project is a continuation of a 2013 GSOC https://bitbucket.org/waedt/wxpython_cffi
 
     TODO: Merge the latest version of the wrappers and finish the sip conversion
 

From pypy.commits at gmail.com  Thu Jun 22 16:24:42 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:42 -0700 (PDT)
Subject: [pypy-commit] pypy issue-2592: a failing test, pop returns None
Message-ID: <594c278a.08aadf0a.65c39.e6ca@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: issue-2592
Changeset: r91631:b0ff2ae9c4b3
Date: 2017-06-22 23:01 +0300
http://bitbucket.org/pypy/pypy/changeset/b0ff2ae9c4b3/

Log:	a failing test, pop returns None

diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -151,6 +151,11 @@
         # tp_as_sequence should be filled, but tp_as_number should be NULL
         assert module.test_tp_as_() == 3
 
+        l = module.newlist()
+        p = l.pop()
+        assert p == 1000
+        assert l == [3, -5]
+
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Thu Jun 22 16:24:46 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:46 -0700 (PDT)
Subject: [pypy-commit] pypy issue-2592: close branch to be merged
Message-ID: <594c278e.2b97df0a.38dad.68c1@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: issue-2592
Changeset: r91633:0aad85e81277
Date: 2017-06-22 23:09 +0300
http://bitbucket.org/pypy/pypy/changeset/0aad85e81277/

Log:	close branch to be merged


From pypy.commits at gmail.com  Thu Jun 22 16:24:44 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:44 -0700 (PDT)
Subject: [pypy-commit] pypy issue-2592: pop and pop_end return values
Message-ID: <594c278c.90b81c0a.3dd79.d86c@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: issue-2592
Changeset: r91632:7fc61a836a84
Date: 2017-06-22 23:09 +0300
http://bitbucket.org/pypy/pypy/changeset/7fc61a836a84/

Log:	pop and pop_end return values

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -346,11 +346,11 @@
 
     def pop(self, w_list, index):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop(w_list, index)
+        return w_list.strategy.pop(w_list, index)
 
     def pop_end(self, w_list):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop_end(w_list)
+        return w_list.strategy.pop_end(w_list)
 
     def insert(self, w_list, index, w_item):
         w_list.switch_to_object_strategy()
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -154,7 +154,9 @@
         l = module.newlist()
         p = l.pop()
         assert p == 1000
-        assert l == [3, -5]
+        p = l.pop(0)
+        assert p == 3
+        assert l == [-5]
 
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""

From pypy.commits at gmail.com  Thu Jun 22 16:24:48 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:48 -0700 (PDT)
Subject: [pypy-commit] pypy default: fix issue #2592 - cpyext list.pop,
 pop_end must return a value
Message-ID: <594c2790.53061c0a.ffcdc.f5e9@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91634:ee2e575b6008
Date: 2017-06-22 23:11 +0300
http://bitbucket.org/pypy/pypy/changeset/ee2e575b6008/

Log:	fix issue #2592 - cpyext list.pop, pop_end must return a value

diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -346,11 +346,11 @@
 
     def pop(self, w_list, index):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop(w_list, index)
+        return w_list.strategy.pop(w_list, index)
 
     def pop_end(self, w_list):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop_end(w_list)
+        return w_list.strategy.pop_end(w_list)
 
     def insert(self, w_list, index, w_item):
         w_list.switch_to_object_strategy()
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -151,6 +151,13 @@
         # tp_as_sequence should be filled, but tp_as_number should be NULL
         assert module.test_tp_as_() == 3
 
+        l = module.newlist()
+        p = l.pop()
+        assert p == 1000
+        p = l.pop(0)
+        assert p == 3
+        assert l == [-5]
+
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [

From pypy.commits at gmail.com  Thu Jun 22 16:24:53 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:53 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix merge
Message-ID: <594c2795.aba0df0a.4ec02.32b6@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r91637:8474e41c4b75
Date: 2017-06-22 23:23 +0300
http://bitbucket.org/pypy/pypy/changeset/8474e41c4b75/

Log:	fix merge

diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -286,7 +286,6 @@
         pvalue[0] = as_pyobj(space, w_value)
     return 1
 
-        _frozendict_cache[space].flag_map_or_seq = 'M'
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def _PyDict_HasOnlyStringKeys(space, w_dict):
     keys_w = space.unpackiterable(w_dict)

From pypy.commits at gmail.com  Thu Jun 22 16:24:49 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: document merged branch
Message-ID: <594c2791.5886df0a.72ef1.9146@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: 
Changeset: r91635:60ad3a7bb257
Date: 2017-06-22 23:13 +0300
http://bitbucket.org/pypy/pypy/changeset/60ad3a7bb257/

Log:	document merged branch

diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -17,3 +17,7 @@
 .. branch: vmprof-0.4.8
 
 Improve and fix issues with vmprof
+
+.. branch: issue-2592
+
+CPyext PyListObject.pop must return the value

From pypy.commits at gmail.com  Thu Jun 22 16:24:51 2017
From: pypy.commits at gmail.com (mattip)
Date: Thu, 22 Jun 2017 13:24:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <594c2793.c385df0a.a2b64.3aa0@mx.google.com>

Author: Matti Picus <matti.picus at gmail.com>
Branch: py3.5
Changeset: r91636:c378444b2cf5
Date: 2017-06-22 23:14 +0300
http://bitbucket.org/pypy/pypy/changeset/c378444b2cf5/

Log:	hg merge default

diff --git a/lib_pypy/cffi/_cffi_errors.h b/lib_pypy/cffi/_cffi_errors.h
--- a/lib_pypy/cffi/_cffi_errors.h
+++ b/lib_pypy/cffi/_cffi_errors.h
@@ -36,7 +36,11 @@
     if (result == NULL)
         goto error;
 
+#if PY_MAJOR_VERSION >= 3
+    bi = PyImport_ImportModule("builtins");
+#else
     bi = PyImport_ImportModule("__builtin__");
+#endif
     if (bi == NULL)
         goto error;
     PyDict_SetItemString(result, "__builtins__", bi);
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -765,7 +765,7 @@
         if sys.platform != "win32":
             return backend.load_library(None, flags)
         name = "c"    # Windows: load_library(None) fails, but this works
-                      # (backward compatibility hack only)
+                      # on Python 2 (backward compatibility hack only)
     first_error = None
     if '.' in name or '/' in name or os.sep in name:
         try:
@@ -775,6 +775,9 @@
     import ctypes.util
     path = ctypes.util.find_library(name)
     if path is None:
+        if name == "c" and sys.platform == "win32" and sys.version_info >= (3,):
+            raise OSError("dlopen(None) cannot work on Windows for Python 3 "
+                          "(see http://bugs.python.org/issue23606)")
         msg = ("ctypes.util.find_library() did not manage "
                "to locate a library called %r" % (name,))
         if first_error is not None:
diff --git a/pypy/doc/project-ideas.rst b/pypy/doc/project-ideas.rst
--- a/pypy/doc/project-ideas.rst
+++ b/pypy/doc/project-ideas.rst
@@ -238,18 +238,17 @@
 using more pypy-friendly technologies, e.g. cffi. Here is a partial list of
 good work that needs to be finished:
 
-**matplotlib** https://github.com/mattip/matplotlib
+**matplotlib** https://github.com/matplotlib/matplotlib
 
-    Status: the repo is an older version of matplotlib adapted to pypy and cpyext
+    TODO: the tkagg backend does not work, which makes tests fail on downstream
+    projects like Pandas, SciPy. It uses id(obj) as a c-pointer to obj in 
+    tkagg.py, which requires refactoring
 
-    TODO: A suggested first step would be to merge the differences into 
-    matplotlib/HEAD. The major problem is the use of a generic view into a
-    numpy ndarray. The int* fields would need to be converted into int[MAX_DIMS]
-    c-arrays and filled in.
+**wxPython** https://bitbucket.org/amauryfa/wxpython-cffi
 
-**wxPython** https://bitbucket.org/waedt/wxpython_cffi
+    Status: A project by a PyPy developer to adapt the Phoenix sip build system to cffi
 
-    Status: A GSOC 2013 project to adapt the Phoenix sip build system to cffi
+    The project is a continuation of a 2013 GSOC https://bitbucket.org/waedt/wxpython_cffi
 
     TODO: Merge the latest version of the wrappers and finish the sip conversion
 
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -17,3 +17,7 @@
 .. branch: vmprof-0.4.8
 
 Improve and fix issues with vmprof
+
+.. branch: issue-2592
+
+CPyext PyListObject.pop must return the value
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -50,6 +50,10 @@
         path = None
     else:
         path = ctypes.util.find_library(name)
+        if path is None and name == 'c':
+            assert sys.platform == 'win32'
+            assert sys.version_info >= (3,)
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     return load_library(path, flags)
 
 def test_load_library():
diff --git a/pypy/module/cpyext/dictobject.py b/pypy/module/cpyext/dictobject.py
--- a/pypy/module/cpyext/dictobject.py
+++ b/pypy/module/cpyext/dictobject.py
@@ -286,6 +286,7 @@
         pvalue[0] = as_pyobj(space, w_value)
     return 1
 
+        _frozendict_cache[space].flag_map_or_seq = 'M'
 @cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)
 def _PyDict_HasOnlyStringKeys(space, w_dict):
     keys_w = space.unpackiterable(w_dict)
diff --git a/pypy/module/cpyext/sequence.py b/pypy/module/cpyext/sequence.py
--- a/pypy/module/cpyext/sequence.py
+++ b/pypy/module/cpyext/sequence.py
@@ -346,11 +346,11 @@
 
     def pop(self, w_list, index):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop(w_list, index)
+        return w_list.strategy.pop(w_list, index)
 
     def pop_end(self, w_list):
         w_list.switch_to_object_strategy()
-        w_list.strategy.pop_end(w_list)
+        return w_list.strategy.pop_end(w_list)
 
     def insert(self, w_list, index, w_item):
         w_list.switch_to_object_strategy()
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -33,6 +33,11 @@
                 return obj;
             '''
             ),
+           ("get_dictproxy", "METH_O",
+            '''
+                return PyDictProxy_New(args);
+            '''
+            ),
            ("check", "METH_O",
             '''
                 return PyLong_FromLong(
@@ -70,6 +75,10 @@
         assert str(e.value).endswith("object is not iterable")
         #
         assert module.check(obj) == 2
+        # make sure dictionaries return false for PySequence_Check
+        assert module.check({'a': 1}) == 2
+        obj = module.get_dictproxy({'a': 10})
+        assert module.check(obj) == 2
 
     def test_iterable_nonmapping_object(self):
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -150,6 +150,13 @@
         # tp_as_sequence should be filled, but tp_as_number should be NULL
         assert module.test_tp_as_() == 3
 
+        l = module.newlist()
+        p = l.pop()
+        assert p == 1000
+        p = l.pop(0)
+        assert p == 3
+        assert l == [-5]
+
     def test_list_macros(self):
         """The PyList_* macros cast, and calls expecting that build."""
         module = self.import_extension('foo', [
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py
@@ -11,6 +11,10 @@
 SIZE_OF_PTR   = ctypes.sizeof(ctypes.c_void_p)
 SIZE_OF_WCHAR = ctypes.sizeof(ctypes.c_wchar)
 
+def needs_dlopen_none():
+    if sys.platform == 'win32' and sys.version_info >= (3,):
+        py.test.skip("dlopen(None) cannot work on Windows for Python 3")
+
 
 class BackendTests:
 
@@ -355,7 +359,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
@@ -938,6 +941,7 @@
     def test_enum_partial(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef(r"enum foo {A, ...}; enum bar { B, C };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.B == 0
         py.test.raises(VerificationMissing, getattr, lib, "A")
@@ -1845,6 +1849,7 @@
             #define DOT_UL 1000UL
             enum foo {AA, BB=DOT, CC};
         """)
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert ffi.string(ffi.cast("enum foo", 100)) == "BB"
         assert lib.DOT_0 == 0
@@ -1874,6 +1879,7 @@
         ffi = FFI()
         ffi.include(ffi1)
         ffi.cdef("enum { EE2, EE3 };")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.EE1 == 0
         assert lib.EE2 == 0
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_function.py
@@ -6,6 +6,7 @@
 from cffi.backend_ctypes import CTypesBackend
 from pypy.module.test_lib_pypy.cffi_tests.udir import udir
 from pypy.module.test_lib_pypy.cffi_tests.support import FdWriteCapture
+from .backend_tests import needs_dlopen_none
 
 try:
     from StringIO import StringIO
@@ -112,6 +113,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -128,6 +130,7 @@
             int fputs(char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ffi.C.fputs   # fetch before capturing, for easier debugging
         with FdWriteCapture() as fd:
@@ -144,6 +147,7 @@
            int fprintf(void *, const char *format, ...);
            void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         with FdWriteCapture() as fd:
             ffi.C.fprintf(ffi.C.stderr, b"hello with no arguments\n")
@@ -170,6 +174,7 @@
         ffi.cdef("""
            int printf(const char *format, ...);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         e = py.test.raises(TypeError, ffi.C.printf, b"hello %d\n", 42)
         assert str(e.value) == ("argument 2 passed in the variadic part "
@@ -180,6 +185,7 @@
         ffi.cdef("""
             int puts(const char *);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.C.puts
         assert ffi.typeof(fptr) == ffi.typeof("int(*)(const char*)")
@@ -203,6 +209,7 @@
             int fputs(const char *, void *);
             void *stderr;
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         fptr = ffi.cast("int(*)(const char *txt, void *)", ffi.C.fputs)
         assert fptr == ffi.C.fputs
@@ -236,6 +243,7 @@
         ffi.cdef("""
             int strlen(char[]);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello")
         res = ffi.C.strlen(p)
@@ -248,6 +256,7 @@
         ffi.cdef("""
             void *stdout;
         """)
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         pout = C.stdout
         C.stdout = ffi.NULL
@@ -260,6 +269,7 @@
         ffi.cdef("""
             char *strchr(const char *s, int c);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         p = ffi.new("char[]", b"hello world!")
         q = ffi.C.strchr(p, ord('w'))
@@ -276,6 +286,7 @@
             struct in_addr { unsigned int s_addr; };
             char *inet_ntoa(struct in_addr in);
         """)
+        needs_dlopen_none()
         ffi.C = ffi.dlopen(None)
         ina = ffi.new("struct in_addr *", [0x04040404])
         a = ffi.C.inet_ntoa(ina[0])
@@ -297,6 +308,7 @@
         filename = str(udir.join('fputs_custom_FILE'))
         ffi = FFI(backend=self.Backend())
         ffi.cdef("int fputs(const char *, FILE *);")
+        needs_dlopen_none()
         C = ffi.dlopen(None)
         with open(filename, 'wb') as f:
             f.write(b'[')
@@ -312,6 +324,7 @@
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""enum foo_e { AA, BB, CC=5, DD };
                     typedef enum { EE=-5, FF } some_enum_t;""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         assert lib.AA == 0
         assert lib.BB == 1
@@ -323,6 +336,7 @@
     def test_void_star_accepts_string(self):
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(const void *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -332,6 +346,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(signed char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
@@ -341,6 +356,7 @@
             py.test.skip("not supported by the ctypes backend")
         ffi = FFI(backend=self.Backend())
         ffi.cdef("""int strlen(unsigned char *);""")
+        needs_dlopen_none()
         lib = ffi.dlopen(None)
         res = lib.strlen(b"hello")
         assert res == 5
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py
@@ -1,6 +1,8 @@
 # Generated by pypy/tool/import_cffi.py
 import py, sys, re
 from cffi import FFI, FFIError, CDefError, VerificationError
+from .backend_tests import needs_dlopen_none
+
 
 class FakeBackend(object):
 
@@ -91,6 +93,7 @@
 def test_pipe():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("int pipe(int pipefd[2]);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.pipe
     assert func.name == 'pipe'
@@ -99,6 +102,7 @@
 def test_vararg():
     ffi = FFI(backend=FakeBackend())
     ffi.cdef("short foo(int, ...);")
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     func = C.foo
     assert func.name == 'foo'
@@ -109,6 +113,7 @@
     ffi.cdef("""
         int foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.foo.BType == '<func (), <int>, False>'
 
@@ -119,6 +124,7 @@
         typedef UInt UIntReally;
         UInt foo(void);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("UIntReally")) == '<unsigned int>'
     assert C.foo.BType == '<func (), <unsigned int>, False>'
@@ -129,6 +135,7 @@
         typedef struct { int a, b; } foo_t, *foo_p;
         int foo(foo_p[]);
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert str(ffi.typeof("foo_t")) == '<int>a, <int>b'
     assert str(ffi.typeof("foo_p")) == '<pointer to <int>a, <int>b>'
@@ -218,6 +225,7 @@
 
 def test_override():
     ffi = FFI(backend=FakeBackend())
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     ffi.cdef("int foo(void);")
     py.test.raises(FFIError, ffi.cdef, "long foo(void);")
@@ -404,6 +412,7 @@
     ffi.cdef("""
         enum Enum { POS = +1, TWO = 2, NIL = 0, NEG = -1, OP = (POS+TWO)-1};
         """)
+    needs_dlopen_none()
     C = ffi.dlopen(None)
     assert C.POS == 1
     assert C.TWO == 2
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py
@@ -419,7 +419,6 @@
         #
         p = ffi.new("wchar_t[]", u+'\U00023456')
         if SIZE_OF_WCHAR == 2:
-            assert sys.maxunicode == 0xffff
             assert len(p) == 3
             assert p[0] == u+'\ud84d'
             assert p[1] == u+'\udc56'
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_re_python.py
@@ -95,6 +95,8 @@
     if sys.platform == 'win32':
         import ctypes.util
         name = ctypes.util.find_msvcrt()
+        if name is None:
+            py.test.skip("dlopen(None) cannot work on Windows with Python 3")
     lib = ffi.dlopen(name)
     assert lib.strlen(b"hello") == 5
 
diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
--- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
+++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py
@@ -2260,6 +2260,9 @@
     assert ffi.typeof("int16_t") is ffi.typeof("char16_t") is ffi.typeof("long")
 
 def test_char16_char32_type(no_cpp=False):
+    if no_cpp is False and sys.platform == "win32":
+        py.test.skip("aaaaaaa why do modern MSVC compilers still define "
+                     "a very old __cplusplus value")
     ffi = FFI()
     ffi.cdef("""
         char16_t foo_2bytes(char16_t);
@@ -2270,7 +2273,7 @@
     typedef uint_least16_t char16_t;
     typedef uint_least32_t char32_t;
     #endif
-    
+
     char16_t foo_2bytes(char16_t a) { return (char16_t)(a + 42); }
     char32_t foo_4bytes(char32_t a) { return (char32_t)(a + 42); }
     """, no_cpp=no_cpp)
diff --git a/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py b/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/test_lib_pypy/cffi_tests/embedding/initerror.py
@@ -0,0 +1,19 @@
+# Generated by pypy/tool/import_cffi.py
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+    int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+    raise KeyError
+""")
+
+ffi.set_source("_initerror_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
+

From pypy.commits at gmail.com  Fri Jun 23 12:19:50 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 23 Jun 2017 09:19:50 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Make slices of array memoryviews usable
 as writable buffers if contiguous.
Message-ID: <594d3fa6.95d91c0a.5bef7.88c7@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91638:9068d4152e3f
Date: 2017-06-23 17:13 +0100
http://bitbucket.org/pypy/pypy/changeset/9068d4152e3f/

Log:	Make slices of array memoryviews usable as writable buffers if
	contiguous.

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -1,5 +1,5 @@
 from rpython.rlib import jit, rgc
-from rpython.rlib.buffer import RawBuffer
+from rpython.rlib.buffer import RawBuffer, SubBuffer
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rlib.rarithmetic import ovfcheck, widen
 from rpython.rlib.unroll import unrolling_iterable
@@ -930,6 +930,14 @@
         assert not self.readonly
         return self.data
 
+    def new_slice(self, start, step, slicelength):
+        if step == 1:
+            n = self.itemsize
+            newbuf = SubBuffer(self.data, start * n, slicelength * n)
+            return ArrayView(newbuf, self.fmt, self.itemsize, self.readonly)
+        else:
+            return BufferView.new_slice(self, start, step, slicelength)
+
 
 unpack_driver = jit.JitDriver(name='unpack_array',
                               greens=['selfclass', 'tp'],
diff --git a/pypy/module/array/test/test_array.py b/pypy/module/array/test/test_array.py
--- a/pypy/module/array/test/test_array.py
+++ b/pypy/module/array/test/test_array.py
@@ -1062,6 +1062,13 @@
     def test_fresh_array_buffer_bytes(self):
         assert bytes(memoryview(self.array('i'))) == b''
 
+    def test_mview_slice_aswritebuf(self):
+        import struct
+        a = self.array('B', b'abcdef')
+        view = memoryview(a)[1:5]
+        struct.pack_into('>H', view, 1, 0x1234)
+        assert a.tobytes() == b'ab\x12\x34ef'
+
 
 class AppTestArrayReconstructor:
     spaceconfig = dict(usemodules=('array', 'struct'))

From pypy.commits at gmail.com  Fri Jun 23 13:26:10 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 23 Jun 2017 10:26:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: remove unittest cruft
Message-ID: <594d4f32.cd871c0a.9cbee.67e4@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91639:51cd15a0c2d6
Date: 2017-06-23 18:25 +0100
http://bitbucket.org/pypy/pypy/changeset/51cd15a0c2d6/

Log:	remove unittest cruft

diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
--- a/extra_tests/test_decimal.py
+++ b/extra_tests/test_decimal.py
@@ -8,52 +8,49 @@
 # import _decimal as C
 # import _pydecimal as P
 
+def test_C():
+    sys.modules["decimal"] = C
+    import decimal
+    d = decimal.Decimal('1')
+    assert isinstance(d, C.Decimal)
+    assert isinstance(d, decimal.Decimal)
+    assert isinstance(d.as_tuple(), C.DecimalTuple)
 
-class TestPythonAPI:
+    assert d == C.Decimal('1')
 
-    def check_equal(self, val, proto):
-        d = C.Decimal(val)
-        p = pickle.dumps(d, proto)
-        assert d == pickle.loads(p)
+def check_round_trip(val, proto):
+    d = C.Decimal(val)
+    p = pickle.dumps(d, proto)
+    assert d == pickle.loads(p)
 
-    def test_C(self):
+def test_pickle():
+    v = '-3.123e81723'
+    for proto in range(pickle.HIGHEST_PROTOCOL + 1):
         sys.modules["decimal"] = C
-        import decimal
-        d = decimal.Decimal('1')
-        assert isinstance(d, C.Decimal)
-        assert isinstance(d, decimal.Decimal)
-        assert isinstance(d.as_tuple(), C.DecimalTuple)
+        check_round_trip('-3.141590000', proto)
+        check_round_trip(v, proto)
 
-        assert d == C.Decimal('1')
+        cd = C.Decimal(v)
+        pd = P.Decimal(v)
+        cdt = cd.as_tuple()
+        pdt = pd.as_tuple()
+        assert cdt.__module__ == pdt.__module__
 
-    def test_pickle(self):
-        v = '-3.123e81723'
-        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
-            sys.modules["decimal"] = C
-            self.check_equal('-3.141590000', proto)
-            self.check_equal(v, proto)
+        p = pickle.dumps(cdt, proto)
+        r = pickle.loads(p)
+        assert isinstance(r, C.DecimalTuple)
+        assert cdt == r
 
-            cd = C.Decimal(v)
-            pd = P.Decimal(v)
-            cdt = cd.as_tuple()
-            pdt = pd.as_tuple()
-            assert cdt.__module__ == pdt.__module__
+        sys.modules["decimal"] = C
+        p = pickle.dumps(cd, proto)
+        sys.modules["decimal"] = P
+        r = pickle.loads(p)
+        assert isinstance(r, P.Decimal)
+        assert r == pd
 
-            p = pickle.dumps(cdt, proto)
-            r = pickle.loads(p)
-            assert isinstance(r, C.DecimalTuple)
-            assert cdt == r
-
-            sys.modules["decimal"] = C
-            p = pickle.dumps(cd, proto)
-            sys.modules["decimal"] = P
-            r = pickle.loads(p)
-            assert isinstance(r, P.Decimal)
-            assert r == pd
-
-            sys.modules["decimal"] = C
-            p = pickle.dumps(cdt, proto)
-            sys.modules["decimal"] = P
-            r = pickle.loads(p)
-            assert isinstance(r, P.DecimalTuple)
-            assert r == pdt
+        sys.modules["decimal"] = C
+        p = pickle.dumps(cdt, proto)
+        sys.modules["decimal"] = P
+        r = pickle.loads(p)
+        assert isinstance(r, P.DecimalTuple)
+        assert r == pdt

From pypy.commits at gmail.com  Fri Jun 23 13:50:37 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 23 Jun 2017 10:50:37 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Use correct implementation for
 Decimal.compare_total{, _mag}(). Fixes #2586 and #2587
Message-ID: <594d54ed.81851c0a.74855.d100@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91640:f2b370578863
Date: 2017-06-23 18:49 +0100
http://bitbucket.org/pypy/pypy/changeset/f2b370578863/

Log:	Use correct implementation for Decimal.compare_total{,_mag}(). Fixes
	#2586 and #2587

diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
--- a/extra_tests/test_decimal.py
+++ b/extra_tests/test_decimal.py
@@ -1,3 +1,5 @@
+import pytest
+
 import pickle
 import sys
 
@@ -8,6 +10,11 @@
 # import _decimal as C
 # import _pydecimal as P
 
+ at pytest.yield_fixture(params=[C, P], ids=['_decimal', '_pydecimal'])
+def module(request):
+    yield request.param
+
+
 def test_C():
     sys.modules["decimal"] = C
     import decimal
@@ -54,3 +61,10 @@
         r = pickle.loads(p)
         assert isinstance(r, P.DecimalTuple)
         assert r == pdt
+
+def test_compare_total(module):
+    assert module.Decimal('12').compare_total(module.Decimal('12.0')) == 1
+    assert module.Decimal('4367').compare_total(module.Decimal('NaN')) == -1
+
+def test_compare_total_mag(module):
+    assert module.Decimal(1).compare_total_mag(-2) == -1
diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -719,8 +719,8 @@
 
     compare = _make_binary_operation('compare')
     compare_signal = _make_binary_operation('compare_signal')
-    compare_total = _make_binary_operation('compare')
-    compare_total_mag = _make_binary_operation('compare')
+    compare_total = _make_binary_operation('compare_total')
+    compare_total_mag = _make_binary_operation('compare_total_mag')
     logical_and = _make_binary_operation('logical_and')
     logical_or = _make_binary_operation('logical_or')
     logical_xor = _make_binary_operation('logical_xor')

From pypy.commits at gmail.com  Fri Jun 23 13:57:43 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Jun 2017 10:57:43 -0700 (PDT)
Subject: [pypy-commit] cffi default: Expand this paragraph
Message-ID: <594d5697.a494df0a.9d2c3.024e@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r2990:0c3756872e68
Date: 2017-06-23 19:57 +0200
http://bitbucket.org/cffi/cffi/changeset/0c3756872e68/

Log:	Expand this paragraph

diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -510,16 +510,26 @@
    ``static char *const FOO;``).
 
 Currently, it is not supported to find automatically which of the
-various integer or float types you need at which place.
-If a type is named, and an integer type, then use ``typedef
-int... the_type_name;``.  In the case of
-function arguments or return type, when it is a simple integer/float
-type, it may be misdeclared (if you misdeclare a function ``void
-f(long)`` as ``void f(int)``, it still works, but you have to call it
-with arguments that fit an int).  But it doesn't work any longer for
-more complex types (e.g. you cannot misdeclare a ``int *`` argument as
-``long *``) or in other locations (e.g. a global array ``int a[5];``
-must not be misdeclared ``long a[5];``).  CFFI considers `all types listed
+various integer or float types you need at which place---except in the
+following case: if such a type is explicitly named.  For an integer
+type, use ``typedef int... the_type_name;``, or another type like
+``typedef unsigned long... the_type_name;``.  Both are equivalent and
+replaced by the real C type, which must be an integer type.
+Similarly, for floating-point types, use ``typedef float...
+the_type_name;`` or equivalently ``typedef double...  the_type_name;``.
+Note that ``long double`` cannot be detected this way.
+
+In the case of function arguments or return types, when it is a simple
+integer/float type, you can simply misdeclare it.  If you misdeclare a
+function ``void f(long)`` as ``void f(int)``, it still works (but you
+have to call it with arguments that fit an int).  It works because the C
+compiler will do the casting for us.  This C-level casting of arguments
+and return types only works for regular function, and not for function
+pointer types; currently, it also does not work for variadic functions.
+
+For more complex types, you have no choice but be precise.  For example,
+you cannot misdeclare a ``int *`` argument as ``long *``, or a global
+array ``int a[5];`` as ``long a[5];``.  CFFI considers `all types listed
 above`_ as primitive (so ``long long a[5];`` and ``int64_t a[5]`` are
 different declarations).  The reason for that is detailed in `a comment
 about an issue.`__

From pypy.commits at gmail.com  Fri Jun 23 18:27:16 2017
From: pypy.commits at gmail.com (arigo)
Date: Fri, 23 Jun 2017 15:27:16 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <594d95c4.103a1c0a.c86ce.be0a@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r893:99c987848217
Date: 2017-06-24 00:27 +0200
http://bitbucket.org/pypy/pypy.org/changeset/99c987848217/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 63.7
+      value: 63.8
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $66891 of $105000 (63.7%)
+   $67020 of $105000 (63.8%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2688 left
+    we have $2804 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Fri Jun 23 18:54:23 2017
From: pypy.commits at gmail.com (wlav)
Date: Fri, 23 Jun 2017 15:54:23 -0700 (PDT)
Subject: [pypy-commit] pypy default: add script to create PyPI package from
 cppyy and stripped-down ROOT/Cling sources
Message-ID: <594d9c1f.0e8b1c0a.6b846.dcc9@mx.google.com>

Author: Wim Lavrijsen <WLavrijsen at lbl.gov>
Branch: 
Changeset: r91641:a4013f68eff6
Date: 2017-06-23 15:42 -0700
http://bitbucket.org/pypy/pypy/changeset/a4013f68eff6/

Log:	add script to create PyPI package from cppyy and stripped-down
	ROOT/Cling sources

diff --git a/pypy/module/cppyy/backend/create_cppyy_package.py b/pypy/module/cppyy/backend/create_cppyy_package.py
new file mode 100755
--- /dev/null
+++ b/pypy/module/cppyy/backend/create_cppyy_package.py
@@ -0,0 +1,649 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import os, sys
+import argparse, re, shutil, tarfile, urllib2
+
+
+DEBUG_TESTBUILD = False
+
+TARBALL_CACHE_DIR = 'releases'
+
+ROOT_KEEP = ['build', 'cmake', 'config', 'core', 'etc', 'interpreter',
+             'io', 'LICENSE', 'net', 'Makefile', 'CMakeLists.txt', 'math',
+             'main'] # main only needed in more recent root b/c of rootcling
+ROOT_CORE_KEEP = ['CMakeLists.txt', 'base', 'clib', 'clingutils', 'cont',
+                  'dictgen', 'foundation', 'lzma', 'macosx', 'meta',
+                  'metacling', 'metautils', 'rootcling_stage1', 'textinput',
+                  'thread', 'unix', 'utils', 'winnt', 'zip']
+ROOT_IO_KEEP = ['CMakeLists.txt', 'io', 'rootpcm']
+ROOT_NET_KEEP = ['CMakeLists.txt', 'net']
+ROOT_MATH_KEEP = ['CMakeLists.txt', 'mathcore']
+ROOT_ETC_KEEP = ['Makefile.arch', 'class.rules', 'cmake', 'dictpch',
+                 'gdb-backtrace.sh', 'gitinfo.txt', 'helgrind-root.supp',
+                 'hostcert.conf', 'system.plugins-ios',
+                 'valgrind-root-python.supp', 'valgrind-root.supp', 'vmc']
+
+ROOT_EXPLICIT_REMOVE = ['core/base/v7', 'math/mathcore/v7', 'io/io/v7']
+
+
+ERR_RELEASE_NOT_FOUND = 2
+
+
+#
+## CLI arguments
+#
+class ReleaseValidation(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        if not re.match(r'6\.\d\d\.\d\d', value):
+            raise argparse.ArgumentTypeError(
+                "release number should of the form '6.dd.dd'")
+        setattr(namespace, self.dest, value)
+        return value
+
+parser = argparse.ArgumentParser(
+    description='Build PyPi package for cppyy containing the minimum of ROOT')
+parser.add_argument('-r', '--release', type=str, nargs='?',
+                    action=ReleaseValidation, help='ROOT release to use')
+
+args = parser.parse_args()
+
+
+#
+## ROOT source pull and cleansing
+#
+def clean_directory(directory, keeplist, trim_cmake=True):
+    removed_entries = []
+    for entry in os.listdir(directory):
+        if entry[0] == '.' or entry in keeplist:
+            continue
+        removed_entries.append(entry)
+        entry = os.path.join(directory, entry)
+        print('now removing', entry)
+        if os.path.isdir(entry):
+            shutil.rmtree(entry)
+        else:
+            os.remove(entry)
+
+    if not trim_cmake:
+        return
+
+    # now take the removed entries out of the CMakeLists.txt
+    if removed_entries:
+        inp = os.path.join(directory, 'CMakeLists.txt')
+        print('trimming', inp)
+        outp = inp+'.new'
+        new_cml = open(outp, 'w')
+        for line in open(inp).readlines():
+            if ('add_subdirectory' in line) or\
+               ('COMMAND' in line and 'copy' in line) or\
+               ('ROOT_ADD_TEST_SUBDIRECTORY' in line) or\
+               ('install(DIRECTORY' in line):
+                for sub in removed_entries:
+                    if sub in line:
+                        line = '#'+line
+                        break
+            new_cml.write(line)
+        new_cml.close()
+        os.rename(outp, inp)
+    else:
+        print('reusing existing %s/CMakeLists.txt' % (directory,))
+ 
+
+class ReleaseValidation(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        if not re.match(r'6\.\d\d\.\d\d', value):
+            raise argparse.ArgumentTypeError(
+                "release number should of the form '6.dd.dd'")
+        setattr(namespace, self.dest, value)
+        return value
+
+parser = argparse.ArgumentParser(
+    description='Build PyPi package for cppyy containing the minimum of ROOT')
+parser.add_argument('-r', '--release', type=str, nargs='?',
+                    action=ReleaseValidation, help='ROOT release to use')
+
+args = parser.parse_args()
+
+if not os.path.exists(TARBALL_CACHE_DIR):
+    os.mkdir(TARBALL_CACHE_DIR)
+
+if args.release:
+  # use provided release
+    fn = 'root_v%s.source.tar.gz' % args.release
+    addr = 'https://root.cern.ch/download/'+fn
+    if not os.path.exists(os.path.join(TARBALL_CACHE_DIR, fn)):
+        try:
+            print('retrieving', fn)
+            resp = urllib2.urlopen(addr, fn)
+            out = open(os.path.join(TARBALL_CACHE_DIR, fn), 'wb')
+            out.write(resp.read())
+            out.close()
+        except urllib2.HTTPError:
+            print('release %s not found' % args.release)
+            sys.exit(ERR_RELEASE_NOT_FOUND)
+    else:
+        print('reusing', fn, 'from local directory')
+else:
+    print('provide release ... getting latest release is not yet implemented ...')
+    sys.exit(1)
+  # get latest and set fn, args.release, etc.
+
+# construct version for package
+args.version = ''
+testnext = False
+for c in args.release:
+    if testnext:
+        testnext = False
+        if c == '0':
+            continue
+    if c == '.':
+        testnext = True
+    args.version += c
+args.version += '.0'
+
+fn = os.path.join(TARBALL_CACHE_DIR, fn)
+pkgdir = os.path.join('root-'+args.release)
+if not os.path.exists(pkgdir):
+    print('now extracting', args.release)
+    tf = tarfile.TarFile.gzopen(fn)
+    tf.extractall()
+    tf.close()
+else:
+    print('reusing existing directory', pkgdir)
+
+# remove everything except for the listed set of libraries
+os.chdir(pkgdir)
+
+clean_directory(os.path.curdir, ROOT_KEEP)
+clean_directory('core',         ROOT_CORE_KEEP)
+clean_directory('etc',          ROOT_ETC_KEEP, trim_cmake=False)
+clean_directory('io',           ROOT_IO_KEEP)
+clean_directory('math',         ROOT_MATH_KEEP)
+clean_directory('net',          ROOT_NET_KEEP)
+
+
+# trim main (only need rootcling)
+print('trimming main')
+for entry in os.listdir('main/src'):
+    if entry != 'rootcling.cxx':
+        os.remove('main/src/'+entry)
+inp = 'main/CMakeLists.txt'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if ('ROOT_EXECUTABLE' in line or\
+        'SET_TARGET_PROPERTIES' in line) and\
+       not 'rootcling' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+
+# remove afterimage and ftgl explicitly
+print('trimming externals')
+for cmf in ['AfterImage', 'FTGL']:
+    os.remove('cmake/modules/Find%s.cmake' % (cmf,))
+inp = 'cmake/modules/SearchInstalledSoftware.cmake'
+outp = inp+'.new'
+now_stripping = False
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if '#---Check for ftgl if needed' == line[0:28] or\
+       '#---Check for AfterImage' == line[0:24]:
+        now_stripping = True
+    elif '#---Check' == line[0:9]:
+        now_stripping = False
+    if now_stripping:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+inp = 'cmake/modules/RootBuildOptions.cmake'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if 'ROOT_BUILD_OPTION(builtin_ftgl' in line or\
+       'ROOT_BUILD_OPTION(builtin_afterimage' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+
+# remove testing and examples
+print('trimming testing')
+inp = 'CMakeLists.txt'
+outp = inp+'.new'
+now_stripping = False
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if '#---Configure Testing using CTest' == line[0:33] or\
+       '#---hsimple.root' == line[0:16]:
+        now_stripping = True
+    elif '#---Packaging' == line[0:13] or\
+         '#---version' == line[0:11]:
+        now_stripping = False
+    if now_stripping:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+print('trimming RootCPack')
+inp = 'cmake/modules/RootCPack.cmake'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp):
+    if 'README.txt' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+# some more explicit removes:
+for dir_to_remove in ROOT_EXPLICIT_REMOVE:
+    try:
+        shutil.rmtree(dir_to_remove)
+    except OSError:
+        pass
+
+# special fixes
+inp = 'core/base/src/TVirtualPad.cxx'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp):
+    if '#include "X3DBuffer.h"' == line[0:22]:
+        line = """//#include "X3DBuffer.h"
+typedef struct _x3d_sizeof_ {
+   int  numPoints;
+   int  numSegs;
+   int  numPolys;
+} Size3D;
+"""
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+inp = 'math/mathcore/src/Fitter.cxx'
+if os.path.exists(inp):
+    outp = inp+'.new'
+    new_cml = open(outp, 'w')
+    for line in open(inp):
+        if '#include "TF1.h"' in line:
+            continue
+        new_cml.write(line)
+    new_cml.close()
+    os.rename(outp, inp)
+
+# done
+os.chdir(os.path.pardir)
+
+# debugging: run a test build
+if DEBUG_TESTBUILD:
+    print('running a debug test build')
+    tb = "test_builddir"
+    if os.path.exists(tb):
+        shutil.rmtree(tb)
+    os.mkdir(tb)
+    os.chdir(tb)
+    os.system('cmake ../%s -DCMAKE_INSTALL_PREFIX=../install -Dminimal=ON -Dasimage=OFF' % pkgdir)
+    os.system('make -j 32')
+
+
+#
+## package creation
+#
+countdown = 0
+pidir = 'Package-'+args.release
+print('creating package', pidir)
+if not os.path.exists(pidir):
+    os.mkdir(pidir)
+os.chdir(pidir); countdown += 1
+
+print('creating LICENSE.txt')
+with open('LICENSE.txt', 'w') as outp:
+    outp.write("""There are three main parts:
+
+ LLVM: distributed under University of Illinois/NCSA Open Source License
+   https://opensource.org/licenses/UoI-NCSA.php
+ ROOT: distributed under LGPL 2.1
+   https://root.cern.ch/license
+ Cppyy: distributed under LBNL BSD
+   https://fedoraproject.org/wiki/Licensing/LBNLBSD
+""")
+
+print('creating MANIFEST.in')
+with open('MANIFEST.in', 'w') as outp:
+    outp.write("""# Include the license file
+include LICENSE.txt
+
+# Include the data files
+recursive-include src *
+""")
+
+print('creating README.rst')
+with open('README.rst', 'w') as outp:
+    outp.write("""PyPy cling-support
+==================
+
+----
+
+Find the documentation here:
+  http://doc.pypy.org/en/latest/cppyy.html
+""")
+
+print('creating setup.cfg')
+with open('setup.cfg', 'w') as outp:
+    outp.write("""[bdist_wheel]
+universal=0
+""")
+
+print('creating setup.py')
+with open('setup.py', 'w') as outp:
+    outp.write("""import os, sys, subprocess
+from setuptools import setup, find_packages
+from distutils import log
+from distutils.command.build import build as _build
+from setuptools.command.install import install as _install
+from distutils.sysconfig import get_python_lib
+from distutils.errors import DistutilsSetupError
+from codecs import open
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
+    long_description = f.read()
+
+builddir = None
+def get_builddir():
+    global builddir
+    if builddir is None:
+        topdir = os.getcwd()
+        builddir = os.path.join(topdir, 'builddir')
+    return builddir
+
+srcdir = None
+def get_srcdir():
+    global srcdir
+    if srcdir is None:
+        topdir = os.getcwd()
+        srcdir = os.path.join(topdir, 'src', 'backend')
+    return srcdir
+
+class my_cmake_build(_build):
+    def __init__(self, dist, *args, **kwargs):
+        _build.__init__(self, dist, *args, **kwargs)
+        # TODO: can't seem to find a better way of getting hold of
+        # the install_lib parameter during the build phase ...
+        prefix = ''
+        try:
+            prefix = dist.get_command_obj('install').install_lib
+        except AttributeError:
+            pass
+        if not prefix:
+            prefix = get_python_lib(1, 0)
+        self.prefix = os.path.join(prefix, 'cppyy_backend')
+
+    def run(self):
+        # base run
+        _build.run(self)
+
+        # custom run
+        log.info('Now building libcppyy_backend.so and dependencies')
+        builddir = get_builddir()
+        srcdir = get_srcdir()
+        if not os.path.exists(builddir):
+            log.info('Creating build directory %s ...' % builddir)
+            os.makedirs(builddir)
+
+        os.chdir(builddir)
+        log.info('Running cmake for cppyy_backend')
+        if subprocess.call([
+                'cmake', srcdir, '-Dminimal=ON -Dasimage=OFF',
+                '-DCMAKE_INSTALL_PREFIX='+self.prefix]) != 0:
+            raise DistutilsSetupError('Failed to configure cppyy_backend')
+
+        nprocs = os.getenv("MAKE_NPROCS")
+        if nprocs:
+            try:
+                ival = int(nprocs)
+                nprocs = '-j'+nprocs
+            except ValueError:
+                log.warn("Integer expected for MAKE_NPROCS, but got %s (ignored)", nprocs)
+                nprocs = '-j1'
+        else:
+            nprocs = '-j1'
+        log.info('Now building cppyy_backend and dependencies ...')
+        if subprocess.call(['make', nprocs]) != 0:
+            raise DistutilsSetupError('Failed to build cppyy_backend')
+
+        log.info('build finished')
+
+class my_libs_install(_install):
+    def run(self):
+        # base install
+        _install.run(self)
+
+        # custom install
+        log.info('Now installing libcppyy_backend.so and dependencies')
+        builddir = get_builddir()
+        if not os.path.exists(builddir):
+            raise DistutilsSetupError('Failed to find build dir!')
+        os.chdir(builddir)
+
+        prefix = self.install_lib
+        log.info('Now installing in %s ...', prefix)
+        if subprocess.call(['make', 'install']) != 0:
+            raise DistutilsSetupError('Failed to install cppyy_backend')
+
+        log.info('install finished')
+
+    def get_outputs(self):
+        outputs = _install.get_outputs(self)
+        outputs.append(os.path.join(self.install_lib, 'cppyy_backend'))
+        return outputs
+
+setup(
+    name='PyPy-cppyy-backend',
+""")
+    outp.write("    version='%s', # corresponds to ROOT %s, extra number is for packager\n"\
+         % (args.version, args.release))
+    outp.write("""    description='Cling support for PyPy',
+    long_description=long_description,
+
+    url='http://pypy.org',
+
+    # Author details
+    author='PyPy Developers',
+    author_email='pypy-dev at python.org',
+
+    license='LLVM: UoI-NCSA; ROOT: LGPL 2.1; Cppyy: LBNL BSD',
+
+    classifiers=[
+        'Development Status :: 4 - Beta',
+
+        'Intended Audience :: Developers',
+
+        'Topic :: Software Development',
+        'Topic :: Software Development :: Interpreters',
+
+        #'License :: OSI Approved :: MIT License',
+
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: PyPy',
+        'Programming Language :: C',
+        'Programming Language :: C++',
+
+        'Natural Language :: English'
+    ],
+
+    keywords='interpreter development',
+
+    packages=find_packages('src', ['backend']),
+    include_package_data=True,
+
+    extras_require={
+    },
+
+    cmdclass = {
+        'build': my_cmake_build,
+        'install': my_libs_install,
+    },
+)
+""")
+
+
+print('creating src ... ROOT part')
+if not os.path.exists('src'):
+    os.mkdir('src')
+os.chdir('src'); countdown += 1
+if not os.path.exists('backend'):
+    src = os.path.join(os.path.pardir, os.path.pardir, pkgdir)
+    print('now copying', src)
+    shutil.copytree(src, 'backend')
+
+print('creating src ... cppyy part')
+os.chdir('backend'); countdown += 1
+if not os.path.exists('cppyy'):
+    os.mkdir('cppyy')
+    os.chdir('cppyy'); countdown += 1
+
+    with open('CMakeLists.txt', 'w') as outp:
+        outp.write("""############################################################################
+# CMakeLists.txt file for building cppyy package
+############################################################################
+
+ROOT_GLOB_SOURCES(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cxx)
+set_source_files_properties(${sources} COMPILE_FLAGS "-fomit-frame-pointer -fvisibility=hidden -DRPY_EXTERN=RPY_EXPORTED -DRPYTHON_LL2CTYPES")
+
+add_definitions(${CLING_CXXFLAGS})
+
+ROOT_LINKER_LIBRARY(cppyy_backend ${sources}
+                               LIBRARIES ${CMAKE_DL_LIBS}
+                               DEPENDENCIES Core Cling RIO Thread)
+
+add_dependencies(cppyy_backend CLING)
+""")
+
+    os.mkdir('src')
+    os.chdir('src'); countdown += 1
+    print('pulling cppyy/clingcwrapper.cxx from pypy')
+    base = 'https://bitbucket.org/pypy/pypy/raw/default/pypy/module/cppyy/'
+    for cppyy_file in ['src/callcontext.h', 'include/capi.h', 'src/clingcwrapper.cxx',
+                       'include/clingcwrapper.h', 'include/cpp_cppyy.h', 'include/cppyy.h']:
+        resp = urllib2.urlopen(base+cppyy_file)
+        with open(os.path.basename(cppyy_file), 'w') as outp:
+            outp.write(resp.read())
+
+    # fix include
+    inp = 'capi.h'
+    outp = inp+'.new'
+    new_cml = open(outp, 'w')
+    for line in open(inp).readlines():
+        if 'src/precommondefs.h' in line:
+            line = '#include "precommondefs.h"\n'
+        new_cml.write(line)
+    new_cml.close()
+    os.rename(outp, inp)
+
+    with open('precommondefs.h', 'w') as outp:
+        outp.write("""/***** Start of precommondefs.h *****/
+
+/* This is extracted from pyconfig.h from CPython.  It sets the macros
+   that affect the features we get from system include files.
+   It must not #include anything. */
+
+#ifndef __PYPY_PRECOMMONDEFS_H
+#define __PYPY_PRECOMMONDEFS_H
+
+
+/* Define on Darwin to activate all library features */
+#define _DARWIN_C_SOURCE 1
+/* This must be set to 64 on some systems to enable large file support. */
+#define _FILE_OFFSET_BITS 64
+/* Define on Linux to activate all library features */
+#define _GNU_SOURCE 1
+/* This must be defined on some systems to enable large file support. */
+#define _LARGEFILE_SOURCE 1
+/* Define on NetBSD to activate all library features */
+#define _NETBSD_SOURCE 1
+/* Define to activate features from IEEE Stds 1003.1-2001 */
+#ifndef _POSIX_C_SOURCE
+#  define _POSIX_C_SOURCE 200112L
+#endif
+/* Define on FreeBSD to activate all library features */
+#define __BSD_VISIBLE 1
+#define __XSI_VISIBLE 700
+/* Windows: winsock/winsock2 mess */
+#define WIN32_LEAN_AND_MEAN
+#ifdef _WIN64
+   typedef          __int64 Signed;
+   typedef unsigned __int64 Unsigned;
+#  define SIGNED_MIN LLONG_MIN
+#else
+   typedef          long Signed;
+   typedef unsigned long Unsigned;
+#  define SIGNED_MIN LONG_MIN
+#endif
+
+#if !defined(RPY_ASSERT) && !defined(RPY_LL_ASSERT) && !defined(NDEBUG)
+#  define NDEBUG
+#endif
+
+
+/* All functions and global variables declared anywhere should use
+   one of the following attributes:
+
+   RPY_EXPORTED:  the symbol is exported out of libpypy-c.so.
+
+   RPY_EXTERN:    the symbol is not exported out of libpypy-c.so, but
+                  otherwise works like 'extern' by being available to
+                  other C sources.
+
+   static:        as usual, this means the symbol is local to this C file.
+
+   Don't use _RPY_HIDDEN directly.  For tests involving building a custom
+   .so, translator/tool/cbuild.py overrides RPY_EXTERN so that it becomes
+   equal to RPY_EXPORTED.
+
+   Any function or global variable declared with no attribute at all is
+   a bug; please report or fix it.
+*/
+#ifdef __GNUC__
+#  define RPY_EXPORTED extern __attribute__((visibility("default")))
+#  define _RPY_HIDDEN  __attribute__((visibility("hidden")))
+#else
+#  define RPY_EXPORTED extern __declspec(dllexport)
+#  define _RPY_HIDDEN  /* nothing */
+#endif
+#ifndef RPY_EXTERN
+#  define RPY_EXTERN   extern _RPY_HIDDEN
+#endif
+
+
+#endif /* __PYPY_PRECOMMONDEFS_H */
+
+/***** End of precommondefs.h *****/
+""")
+
+# back up to pip package top
+for i in range(countdown-1):
+    os.chdir(os.path.pardir)
+
+# add cppyy module to cmake
+os.chdir('src/backend')
+inp = 'CMakeLists.txt'
+print('adding cppyy to cmake')
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if 'add_subdirectory' in line and 'net' in line:
+        line += 'add_subdirectory (cppyy)\n'
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+# done!

From pypy.commits at gmail.com  Sat Jun 24 06:17:37 2017
From: pypy.commits at gmail.com (stevie_92)
Date: Sat, 24 Jun 2017 03:17:37 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-trialdeletion: Implemented stubs for a
 trial deletion algorithm for cpyext-only reference cycles
Message-ID: <594e3c41.9aa0df0a.21716.c394@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-trialdeletion
Changeset: r91642:e65c2e7763e7
Date: 2017-06-12 20:08 +0200
http://bitbucket.org/pypy/pypy/changeset/e65c2e7763e7/

Log:	Implemented stubs for a trial deletion algorithm for cpyext-only
	reference cycles

diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h
--- a/pypy/module/cpyext/include/object.h
+++ b/pypy/module/cpyext/include/object.h
@@ -26,26 +26,26 @@
 #define PyVarObject_HEAD_INIT(type, size)	\
 	PyObject_HEAD_INIT(type) size,
 
-#ifdef PYPY_DEBUG_REFCOUNT
-/* Slow version, but useful for debugging */
+// #ifdef PYPY_DEBUG_REFCOUNT
+// /* Slow version, but useful for debugging */
 #define Py_INCREF(ob)   (Py_IncRef((PyObject *)(ob)))
 #define Py_DECREF(ob)   (Py_DecRef((PyObject *)(ob)))
 #define Py_XINCREF(ob)  (Py_IncRef((PyObject *)(ob)))
 #define Py_XDECREF(ob)  (Py_DecRef((PyObject *)(ob)))
-#else
-/* Fast version */
-#define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
-#define Py_DECREF(op)                                   \
-    do {                                                \
-        if (--((PyObject *)(op))->ob_refcnt != 0)       \
-            ;                                           \
-        else                                            \
-            _Py_Dealloc((PyObject *)(op));              \
-    } while (0)
+// #else
+// /* Fast version */
+// #define Py_INCREF(ob)   (((PyObject *)(ob))->ob_refcnt++)
+// #define Py_DECREF(op)                                   \
+//     do {                                                \
+//         if (--((PyObject *)(op))->ob_refcnt != 0)       \
+//             ;                                           \
+//         else                                            \
+//             _Py_Dealloc((PyObject *)(op));              \
+//     } while (0)
 
-#define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
-#define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
-#endif
+// #define Py_XINCREF(op) do { if ((op) == NULL) ; else Py_INCREF(op); } while (0)
+// #define Py_XDECREF(op) do { if ((op) == NULL) ; else Py_DECREF(op); } while (0)
+// #endif
 
 #define Py_CLEAR(op)				\
         do {                            	\
diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -318,9 +318,30 @@
             obj.c_ob_refcnt -= 1
             if obj.c_ob_refcnt == 0:
                 _Py_Dealloc(space, obj)
+            else:
+                trial_delete(space, obj)
     else:
-        get_w_obj_and_decref(space, obj)
+        get_w_obj_and_decref(space, obj)  # trial_delete?
 
+ at specialize.ll()
+def trial_delete(space, obj):
+    from pypy.module.cpyext.api import generic_cpy_call, slot_function
+    from pypy.module.cpyext.typeobjectdefs import visitproc
+    from pypy.module.cpyext.slotdefs import llslot
+
+    if not obj.c_ob_type or not obj.c_ob_type.c_tp_traverse:
+        return
+
+    @slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+    def visit(space, obj, args):
+        w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
+        print "visit", obj, w_type.name
+        return 0
+
+    print "trial_delete", obj, obj.c_ob_refcnt
+
+    proc = rffi.cast(visitproc, llslot(space, visit))
+    generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, proc, None)
 
 @cpython_api([PyObject], lltype.Void)
 def Py_IncRef(space, obj):
diff --git a/pypy/module/cpyext/test/test_cpyext_gc.py b/pypy/module/cpyext/test/test_cpyext_gc.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_cpyext_gc.py
@@ -0,0 +1,597 @@
+import sys
+import weakref
+
+import pytest
+
+from pypy.tool.cpyext.extbuild import (
+    SystemCompilationInfo, HERE, get_sys_info_app)
+from pypy.interpreter.gateway import unwrap_spec, interp2app
+from rpython.rtyper.lltypesystem import lltype, ll2ctypes
+from pypy.module.cpyext import api
+from pypy.module.cpyext.state import State
+from pypy.module.cpyext.pyobject import Py_DecRef
+from rpython.tool.identity_dict import identity_dict
+from rpython.tool import leakfinder
+from rpython.rlib import rawrefcount
+from rpython.tool.udir import udir
+
+only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names"
+
+ at api.cpython_api([], api.PyObject)
+def PyPy_Crash1(space):
+    1/0
+
+ at api.cpython_api([], lltype.Signed, error=-1)
+def PyPy_Crash2(space):
+    1/0
+
+class SpaceCompiler(SystemCompilationInfo):
+    """Extension compiler for regular (untranslated PyPy) mode"""
+    def __init__(self, space, *args, **kwargs):
+        self.space = space
+        SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+    def load_module(self, mod, name):
+        space = self.space
+        api.load_extension_module(space, mod, name)
+        return space.getitem(
+            space.sys.get('modules'), space.wrap(name))
+
+
+def get_cpyext_info(space):
+    from pypy.module.imp.importing import get_so_extension
+    state = space.fromcache(State)
+    api_library = state.api_lib
+    if sys.platform == 'win32':
+        libraries = [api_library]
+        # '%s' undefined; assuming extern returning int
+        compile_extra = ["/we4013"]
+        # prevent linking with PythonXX.lib
+        w_maj, w_min = space.fixedview(space.sys.get('version_info'), 5)[:2]
+        link_extra = ["/NODEFAULTLIB:Python%d%d.lib" %
+            (space.int_w(w_maj), space.int_w(w_min))]
+    else:
+        libraries = []
+        if sys.platform.startswith('linux'):
+            compile_extra = [
+                "-Werror", "-g", "-O0", "-Wp,-U_FORTIFY_SOURCE", "-fPIC"]
+            link_extra = ["-g"]
+        else:
+            compile_extra = link_extra = None
+    return SpaceCompiler(space,
+        builddir_base=udir,
+        include_extra=api.include_dirs,
+        compile_extra=compile_extra,
+        link_extra=link_extra,
+        extra_libs=libraries,
+        ext=get_so_extension(space))
+
+
+def freeze_refcnts(self):
+    rawrefcount._dont_free_any_more()
+    return #ZZZ
+    state = self.space.fromcache(RefcountState)
+    self.frozen_refcounts = {}
+    for w_obj, obj in state.py_objects_w2r.iteritems():
+        self.frozen_refcounts[w_obj] = obj.c_ob_refcnt
+    #state.print_refcounts()
+    self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
+
+class LeakCheckingTest(object):
+    """Base class for all cpyext tests."""
+    spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array',
+                                   'itertools', 'time', 'binascii',
+                                   'micronumpy', 'mmap'
+                                   ])
+
+    enable_leak_checking = True
+
+    @staticmethod
+    def cleanup_references(space):
+        return #ZZZ
+        state = space.fromcache(RefcountState)
+
+        import gc; gc.collect()
+        # Clear all lifelines, objects won't resurrect
+        for w_obj, obj in state.lifeline_dict._dict.items():
+            if w_obj not in state.py_objects_w2r:
+                state.lifeline_dict.set(w_obj, None)
+            del obj
+        import gc; gc.collect()
+
+
+        for w_obj in state.non_heaptypes_w:
+            Py_DecRef(space, w_obj)
+        state.non_heaptypes_w[:] = []
+        state.reset_borrowed_references()
+
+    def check_and_print_leaks(self):
+        rawrefcount._collect()
+        # check for sane refcnts
+        import gc
+
+        if 1:  #ZZZ  not self.enable_leak_checking:
+            leakfinder.stop_tracking_allocations(check=False)
+            return False
+
+        leaking = False
+        state = self.space.fromcache(RefcountState)
+        gc.collect()
+        lost_objects_w = identity_dict()
+        lost_objects_w.update((key, None) for key in self.frozen_refcounts.keys())
+
+        for w_obj, obj in state.py_objects_w2r.iteritems():
+            base_refcnt = self.frozen_refcounts.get(w_obj)
+            delta = obj.c_ob_refcnt
+            if base_refcnt is not None:
+                delta -= base_refcnt
+                lost_objects_w.pop(w_obj)
+            if delta != 0:
+                leaking = True
+                print >>sys.stderr, "Leaking %r: %i references" % (w_obj, delta)
+                try:
+                    weakref.ref(w_obj)
+                except TypeError:
+                    lifeline = None
+                else:
+                    lifeline = state.lifeline_dict.get(w_obj)
+                if lifeline is not None:
+                    refcnt = lifeline.pyo.c_ob_refcnt
+                    if refcnt > 0:
+                        print >>sys.stderr, "\tThe object also held by C code."
+                    else:
+                        referrers_repr = []
+                        for o in gc.get_referrers(w_obj):
+                            try:
+                                repr_str = repr(o)
+                            except TypeError as e:
+                                repr_str = "%s (type of o is %s)" % (str(e), type(o))
+                            referrers_repr.append(repr_str)
+                        referrers = ", ".join(referrers_repr)
+                        print >>sys.stderr, "\tThe object is referenced by these objects:", \
+                                referrers
+        for w_obj in lost_objects_w:
+            print >>sys.stderr, "Lost object %r" % (w_obj, )
+            leaking = True
+        # the actual low-level leak checking is done by pypy.tool.leakfinder,
+        # enabled automatically by pypy.conftest.
+        return leaking
+
+class AppTestApi(LeakCheckingTest):
+    def setup_class(cls):
+        from rpython.rlib.clibffi import get_libc_name
+        if cls.runappdirect:
+            cls.libc = get_libc_name()
+        else:
+            cls.w_libc = cls.space.wrap(get_libc_name())
+
+    def setup_method(self, meth):
+        if not self.runappdirect:
+            freeze_refcnts(self)
+
+    def teardown_method(self, meth):
+        if self.runappdirect:
+            return
+        self.space.getexecutioncontext().cleanup_cpyext_state()
+        self.cleanup_references(self.space)
+        # XXX: like AppTestCpythonExtensionBase.teardown_method:
+        # find out how to disable check_and_print_leaks() if the
+        # test failed
+        assert not self.check_and_print_leaks(), (
+            "Test leaks or loses object(s).  You should also check if "
+            "the test actually passed in the first place; if it failed "
+            "it is likely to reach this place.")
+
+
+def _unwrap_include_dirs(space, w_include_dirs):
+    if w_include_dirs is None:
+        return None
+    else:
+        return [space.str_w(s) for s in space.listview(w_include_dirs)]
+
+def debug_collect(space):
+    rawrefcount._collect()
+
+class AppTestCpythonExtensionBase(LeakCheckingTest):
+
+    def setup_class(cls):
+        space = cls.space
+        cls.w_here = space.wrap(str(HERE))
+        cls.w_udir = space.wrap(str(udir))
+        cls.w_runappdirect = space.wrap(cls.runappdirect)
+        if not cls.runappdirect:
+            cls.sys_info = get_cpyext_info(space)
+            space.getbuiltinmodule("cpyext")
+            # 'import os' to warm up reference counts
+            w_import = space.builtin.getdictvalue(space, '__import__')
+            space.call_function(w_import, space.wrap("os"))
+            #state = cls.space.fromcache(RefcountState) ZZZ
+            #state.non_heaptypes_w[:] = []
+            cls.w_debug_collect = space.wrap(interp2app(debug_collect))
+        else:
+            def w_import_module(self, name, init=None, body='', filename=None,
+                    include_dirs=None, PY_SSIZE_T_CLEAN=False):
+                from extbuild import get_sys_info_app
+                sys_info = get_sys_info_app(self.udir)
+                return sys_info.import_module(
+                    name, init=init, body=body, filename=filename,
+                    include_dirs=include_dirs,
+                    PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+            cls.w_import_module = w_import_module
+
+            def w_import_extension(self, modname, functions, prologue="",
+                include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+                from extbuild import get_sys_info_app
+                sys_info = get_sys_info_app(self.udir)
+                return sys_info.import_extension(
+                    modname, functions, prologue=prologue,
+                    include_dirs=include_dirs, more_init=more_init,
+                    PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+            cls.w_import_extension = w_import_extension
+
+            def w_compile_module(self, name,
+                    source_files=None, source_strings=None):
+                from extbuild import get_sys_info_app
+                sys_info = get_sys_info_app(self.udir)
+                return sys_info.compile_extension_module(name,
+                    source_files=source_files, source_strings=source_strings)
+            cls.w_compile_module = w_compile_module
+
+            def w_load_module(self, mod, name):
+                from extbuild import get_sys_info_app
+                sys_info = get_sys_info_app(self.udir)
+                return sys_info.load_module(mod, name)
+            cls.w_load_module = w_load_module
+
+            def w_debug_collect(self):
+                import gc
+                gc.collect()
+                gc.collect()
+                gc.collect()
+            cls.w_debug_collect = w_debug_collect
+
+
+    def record_imported_module(self, name):
+        """
+        Record a module imported in a test so that it can be cleaned up in
+        teardown before the check for leaks is done.
+
+        name gives the name of the module in the space's sys.modules.
+        """
+        self.imported_module_names.append(name)
+
+    def setup_method(self, func):
+        if self.runappdirect:
+            return
+
+        @unwrap_spec(name='text')
+        def compile_module(space, name,
+                           w_source_files=None,
+                           w_source_strings=None):
+            """
+            Build an extension module linked against the cpyext api library.
+            """
+            if not space.is_none(w_source_files):
+                source_files = space.listview_bytes(w_source_files)
+            else:
+                source_files = None
+            if not space.is_none(w_source_strings):
+                source_strings = space.listview_bytes(w_source_strings)
+            else:
+                source_strings = None
+            pydname = self.sys_info.compile_extension_module(
+                name,
+                source_files=source_files,
+                source_strings=source_strings)
+
+            # hackish, but tests calling compile_module() always end up
+            # importing the result
+            self.record_imported_module(name)
+
+            return space.wrap(pydname)
+
+        @unwrap_spec(name='text', init='text_or_none', body='text',
+                     filename='fsencode_or_none', PY_SSIZE_T_CLEAN=bool)
+        def import_module(space, name, init=None, body='',
+                          filename=None, w_include_dirs=None,
+                          PY_SSIZE_T_CLEAN=False):
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_module(
+                name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+            self.record_imported_module(name)
+            return w_result
+
+
+        @unwrap_spec(mod='text', name='text')
+        def load_module(space, mod, name):
+            return self.sys_info.load_module(mod, name)
+
+        @unwrap_spec(modname='text', prologue='text',
+                             more_init='text', PY_SSIZE_T_CLEAN=bool)
+        def import_extension(space, modname, w_functions, prologue="",
+                             w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+            functions = space.unwrap(w_functions)
+            include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+            w_result = self.sys_info.import_extension(
+                modname, functions, prologue, include_dirs, more_init,
+                PY_SSIZE_T_CLEAN)
+            self.record_imported_module(modname)
+            return w_result
+
+        # A list of modules which the test caused to be imported (in
+        # self.space).  These will be cleaned up automatically in teardown.
+        self.imported_module_names = []
+
+        wrap = self.space.wrap
+        self.w_compile_module = wrap(interp2app(compile_module))
+        self.w_load_module = wrap(interp2app(load_module))
+        self.w_import_module = wrap(interp2app(import_module))
+        self.w_import_extension = wrap(interp2app(import_extension))
+
+        # create the file lock before we count allocations
+        self.space.call_method(self.space.sys.get("stdout"), "flush")
+
+        freeze_refcnts(self)
+        #self.check_and_print_leaks()
+
+    def unimport_module(self, name):
+        """
+        Remove the named module from the space's sys.modules.
+        """
+        w_modules = self.space.sys.get('modules')
+        w_name = self.space.wrap(name)
+        self.space.delitem(w_modules, w_name)
+
+    def teardown_method(self, func):
+        if self.runappdirect:
+            return
+        for name in self.imported_module_names:
+            self.unimport_module(name)
+        self.space.getexecutioncontext().cleanup_cpyext_state()
+        self.cleanup_references(self.space)
+        # XXX: find out how to disable check_and_print_leaks() if the
+        # test failed...
+        assert not self.check_and_print_leaks(), (
+            "Test leaks or loses object(s).  You should also check if "
+            "the test actually passed in the first place; if it failed "
+            "it is likely to reach this place.")
+
+
+class AppTestCpythonExtension(AppTestCpythonExtensionBase):
+
+    def test_refcount(self):
+        import sys, gc
+        init = """
+        if (Py_IsInitialized()) {
+            PyObject* m;
+
+            if (PyType_Ready(&CycleType) < 0)
+                return;
+
+            m = Py_InitModule3("cycle", module_methods,
+                            "Example module that creates an extension type.");
+
+            if (m == NULL)
+                return;
+
+            Py_INCREF(&CycleType);
+            PyModule_AddObject(m, "Cycle", (PyObject *)&CycleType);
+        }
+        """
+        body = """
+                #include <Python.h>
+                #include "structmember.h"
+
+                typedef struct {
+                    PyObject_HEAD
+                    PyObject *next;
+                    PyObject *val;
+                } Cycle;
+
+                static PyTypeObject CycleType;
+
+                static int
+                Cycle_traverse(Cycle *self, visitproc visit, void *arg)
+                {
+                    int vret;
+
+                    if (self->next) {
+                        vret = visit(self->next, arg);
+                        if (vret != 0)
+                            return vret;
+                    }
+                    if (self->val) {
+                        vret = visit(self->val, arg);
+                        if (vret != 0)
+                            return vret;
+                    }
+
+                    return 0;
+                }
+
+                static int
+                Cycle_clear(Cycle *self)
+                {
+                    PyObject *tmp;
+
+                    tmp = self->next;
+                    self->next = NULL;
+                    Py_XDECREF(tmp);
+
+                    tmp = self->val;
+                    self->val = NULL;
+                    Py_XDECREF(tmp);
+
+                    return 0;
+                }
+
+                static void
+                Cycle_dealloc(Cycle* self)
+                {
+                    Cycle_clear(self);
+                    Py_TYPE(self)->tp_free((PyObject*)self);
+                }
+
+                static PyObject *
+                Cycle_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
+                {
+                    Cycle *self;
+
+                    self = (Cycle *)type->tp_alloc(type, 0);
+                    if (self != NULL) {
+                        self->next = PyString_FromString("");
+                        if (self->next == NULL) {
+                            Py_DECREF(self);
+                            return NULL;
+                        }
+                    }
+
+                    return (PyObject *)self;
+                }
+
+                static int
+                Cycle_init(Cycle *self, PyObject *args, PyObject *kwds)
+                {
+                    PyObject *next=NULL, *tmp;
+
+                    static char *kwlist[] = {"next", NULL};
+
+                    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist,
+                                                    &next))
+                        return -1;
+
+                    if (next) {
+                        tmp = self->next;
+                        Py_INCREF(next);
+                        self->next = next;
+                        Py_XDECREF(tmp);
+                    }
+
+                    return 0;
+                }
+
+                static Cycle *lastCreated;
+
+                static PyObject * Cycle_createCycle(Cycle *self, PyObject *val)
+                {
+                    Cycle *c;
+
+                    c = PyObject_GC_New(Cycle, &CycleType);
+                    if (c == NULL)
+                        return NULL;
+
+                    // set value
+                    Py_INCREF(val);
+                    c->val = val;
+
+                    // track by GC
+                    PyObject_GC_Track(c);
+
+                    // create cycle
+                    Py_INCREF(c);
+                    c->next = (PyObject *)c;
+
+
+                    // save c, but do no INCREF -> reference in lastCreated might become broken
+                    lastCreated = (Cycle *)c;
+
+                    // throw away reference to c
+                    Py_DECREF(c);
+
+                    // return None
+                    Py_INCREF(Py_None);
+                    return Py_None;
+                }
+
+                static PyObject * Cycle_breakCycle(Cycle *self)
+                {
+                    Cycle *tmp;
+
+                    // set next to None
+                    tmp = (Cycle *)lastCreated->next;
+                    Py_INCREF(Py_None);
+                    lastCreated->next = Py_None;
+                    Py_XDECREF(tmp);
+
+                    // return None
+                    Py_INCREF(Py_None);
+                    return Py_None;
+                }
+
+                static PyMemberDef Cycle_members[] = {
+                    {"next", T_OBJECT_EX, offsetof(Cycle, next), 0,
+                    "next"},
+                    {"val", T_OBJECT_EX, offsetof(Cycle, val), 0,
+                    "val"},
+                    {NULL}  /* Sentinel */
+                };
+
+                static PyMethodDef Cycle_methods[] = {
+                    {NULL}  /* Sentinel */
+                };
+
+                static PyTypeObject CycleType = {
+                    PyVarObject_HEAD_INIT(NULL, 0)
+                    "Cycle.Cycle",             /* tp_name */
+                    sizeof(Cycle),             /* tp_basicsize */
+                    0,                         /* tp_itemsize */
+                    (destructor)Cycle_dealloc, /* tp_dealloc */
+                    0,                         /* tp_print */
+                    0,                         /* tp_getattr */
+                    0,                         /* tp_setattr */
+                    0,                         /* tp_compare */
+                    0,                         /* tp_repr */
+                    0,                         /* tp_as_number */
+                    0,                         /* tp_as_sequence */
+                    0,                         /* tp_as_mapping */
+                    0,                         /* tp_hash */
+                    0,                         /* tp_call */
+                    0,                         /* tp_str */
+                    0,                         /* tp_getattro */
+                    0,                         /* tp_setattro */
+                    0,                         /* tp_as_buffer */
+                    Py_TPFLAGS_DEFAULT |
+                        Py_TPFLAGS_BASETYPE |
+                        Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+                    "Cycle objects",           /* tp_doc */
+                    (traverseproc)Cycle_traverse,   /* tp_traverse */
+                    (inquiry)Cycle_clear,           /* tp_clear */
+                    0,                         /* tp_richcompare */
+                    0,                         /* tp_weaklistoffset */
+                    0,                         /* tp_iter */
+                    0,                         /* tp_iternext */
+                    Cycle_methods,             /* tp_methods */
+                    Cycle_members,             /* tp_members */
+                    0,                         /* tp_getset */
+                    0,                         /* tp_base */
+                    0,                         /* tp_dict */
+                    0,                         /* tp_descr_get */
+                    0,                         /* tp_descr_set */
+                    0,                         /* tp_dictoffset */
+                    (initproc)Cycle_init,      /* tp_init */
+                    0,                         /* tp_alloc */
+                    Cycle_new,                 /* tp_new */
+                };
+
+                static PyMethodDef module_methods[] = {
+                    {"breakCycle", (PyCFunction)Cycle_breakCycle, METH_NOARGS, "break cycle"},
+                    {"createCycle", (PyCFunction)Cycle_createCycle, METH_OLDARGS, "create a special cycle"},
+                    {NULL}  /* Sentinel */
+                };
+        """
+        cycle = self.import_module(name='cycle', init=init, body=body)
+
+        class Example(object):
+            def __init__(self):
+                self.next = None
+
+            def __del__(self):
+                print("free")
+
+        cycle.createCycle(Example())
+
+        gc.collect()
+
+        # check if object has been freed
+
+        assert False

From pypy.commits at gmail.com  Sat Jun 24 06:17:39 2017
From: pypy.commits at gmail.com (stevie_92)
Date: Sat, 24 Jun 2017 03:17:39 -0700 (PDT)
Subject: [pypy-commit] pypy cpyext-gc-trialdeletion: Implemented simple
 trial deletion for cpyext-only cycles
Message-ID: <594e3c43.6288df0a.9867a.89b2@mx.google.com>

Author: Stefan Beyer <home at sbeyer.at>
Branch: cpyext-gc-trialdeletion
Changeset: r91643:ca3a43da189f
Date: 2017-06-24 10:56 +0200
http://bitbucket.org/pypy/pypy/changeset/ca3a43da189f/

Log:	Implemented simple trial deletion for cpyext-only cycles

diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py
--- a/pypy/module/cpyext/pyobject.py
+++ b/pypy/module/cpyext/pyobject.py
@@ -15,7 +15,9 @@
 from rpython.rlib.objectmodel import keepalive_until_here
 from rpython.rtyper.annlowlevel import llhelper
 from rpython.rlib import rawrefcount
-from rpython.rlib.debug import fatalerror
+from rpython.rlib.debug import fatalerror, debug_print
+from pypy.module.cpyext.api import slot_function
+from pypy.module.cpyext.typeobjectdefs import visitproc
 
 
 #________________________________________________________
@@ -316,32 +318,106 @@
         if obj:
             assert obj.c_ob_refcnt > 0
             obj.c_ob_refcnt -= 1
-            if obj.c_ob_refcnt == 0:
+            if obj.c_ob_refcnt == 0 and \
+               rawrefcount.get_trialdeletion_phase() != 1:
+                debug_print("dealloc", obj)
                 _Py_Dealloc(space, obj)
+            elif obj.c_ob_refcnt == rawrefcount.REFCNT_FROM_PYPY:
+                debug_print("dead", obj)
             else:
-                trial_delete(space, obj)
+                if rawrefcount.get_trialdeletion_phase() == 0:
+                    trial_delete(space, obj)
     else:
-        get_w_obj_and_decref(space, obj)  # trial_delete?
+        get_w_obj_and_decref(space, obj)
+
+def traverse(space, obj, visit):
+    from pypy.module.cpyext.api import generic_cpy_call
+    if obj.c_ob_type and obj.c_ob_type.c_tp_traverse:
+        generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, visit,
+                         rffi.cast(rffi.VOIDP, obj))
+
+def clear(space, obj):
+    from pypy.module.cpyext.api import generic_cpy_call
+    if obj.c_ob_type:
+        generic_cpy_call(space, obj.c_ob_type.c_tp_clear, obj)
+
+ at slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def visit_decref(space, obj, args):
+    obj.c_ob_refcnt = obj.c_ob_refcnt - 1
+    debug_print("visited dec", obj, "new refcnt", obj.c_ob_refcnt)
+    if (obj not in rawrefcount.get_visited()):
+        rawrefcount.add_visited(obj)
+        from pypy.module.cpyext.slotdefs import llslot
+        traverse(space, obj, rffi.cast(visitproc, llslot(space, visit_decref)))
+    return 0
+
+ at slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
+def visit_incref(space, obj, args):
+    obj.c_ob_refcnt = obj.c_ob_refcnt + 1
+    debug_print("visited inc", obj, "new refcnt", obj.c_ob_refcnt)
+    if (obj not in rawrefcount.get_visited()):
+        rawrefcount.add_visited(obj)
+        from pypy.module.cpyext.slotdefs import llslot
+        traverse(space, obj, rffi.cast(visitproc, llslot(space, visit_incref)))
+    return 0
 
 @specialize.ll()
 def trial_delete(space, obj):
-    from pypy.module.cpyext.api import generic_cpy_call, slot_function
-    from pypy.module.cpyext.typeobjectdefs import visitproc
-    from pypy.module.cpyext.slotdefs import llslot
-
     if not obj.c_ob_type or not obj.c_ob_type.c_tp_traverse:
         return
 
-    @slot_function([PyObject, rffi.VOIDP], rffi.INT_real, error=-1)
-    def visit(space, obj, args):
-        w_type = from_ref(space, rffi.cast(PyObject, obj.c_ob_type))
-        print "visit", obj, w_type.name
-        return 0
+    from pypy.module.cpyext.slotdefs import llslot
+    visitproc_incref = rffi.cast(visitproc, llslot(space, visit_incref))
+    visitproc_decref = rffi.cast(visitproc, llslot(space, visit_decref))
 
-    print "trial_delete", obj, obj.c_ob_refcnt
+    rawrefcount.set_trialdeletion_phase(1)
 
-    proc = rffi.cast(visitproc, llslot(space, visit))
-    generic_cpy_call(space, obj.c_ob_type.c_tp_traverse, obj, proc, None)
+    debug_print("trial_delete", obj, "refct after decref", obj.c_ob_refcnt)
+
+    debug_print("decref phase")
+    rawrefcount.clear_visited()
+    rawrefcount.add_visited(obj)
+    traverse(space, obj, visitproc_decref)
+
+    debug_print("checkref phase")
+    visited = []
+    alive = []
+    for visited_obj in rawrefcount.get_visited():
+        visited.append(visited_obj)
+        if visited_obj.c_ob_refcnt != 0 and \
+           visited_obj.c_ob_refcnt != rawrefcount.REFCNT_FROM_PYPY:
+            alive.append(visited_obj)
+            debug_print("alive", visited_obj)
+
+    debug_print("incref phase")
+    rawrefcount.clear_visited()
+    for alive_obj in alive:
+        if alive_obj not in rawrefcount.get_visited():
+            rawrefcount.add_visited(alive_obj)
+            traverse(space, alive_obj, visitproc_incref)
+
+    alive = []
+    for alive_obj in rawrefcount.get_visited():
+        debug_print("alive", alive_obj, alive_obj.c_ob_refcnt)
+        alive.append(alive_obj)
+
+    for reachable_obj in visited:
+        if reachable_obj not in rawrefcount.get_visited():
+            rawrefcount.add_visited(reachable_obj)
+            traverse(space, reachable_obj, visitproc_incref)
+
+    debug_print("clear phase")
+    rawrefcount.set_trialdeletion_phase(2)
+
+    for reachable_obj in visited:
+        if reachable_obj not in alive:
+            if reachable_obj.c_ob_refcnt < rawrefcount.REFCNT_FROM_PYPY \
+               and reachable_obj.c_ob_refcnt > 0:
+                debug_print("clear", reachable_obj)
+                clear(space, reachable_obj)
+
+    rawrefcount.set_trialdeletion_phase(0)
+    rawrefcount.clear_visited()
 
 @cpython_api([PyObject], lltype.Void)
 def Py_IncRef(space, obj):
@@ -368,6 +444,10 @@
     rawrefcount.mark_deallocating(w_marker_deallocating, obj)
     generic_cpy_call(space, pto.c_tp_dealloc, obj)
 
+ at cpython_api([PyObject], lltype.Void)
+def _Py_Mark(space, obj):
+    rawrefcount.add_marked(obj)
+
 @cpython_api([rffi.VOIDP], lltype.Signed, error=CANNOT_FAIL)
 def _Py_HashPointer(space, ptr):
     return rffi.cast(lltype.Signed, ptr)
diff --git a/pypy/module/cpyext/test/test_cpyext_gc.py b/pypy/module/cpyext/test/test_cpyext_gc.py
--- a/pypy/module/cpyext/test/test_cpyext_gc.py
+++ b/pypy/module/cpyext/test/test_cpyext_gc.py
@@ -356,242 +356,419 @@
             "the test actually passed in the first place; if it failed "
             "it is likely to reach this place.")
 
+def collect(space):
+    import gc
+    rawrefcount._collect()
+    gc.collect()
 
-class AppTestCpythonExtension(AppTestCpythonExtensionBase):
+class AppTestCpythonExtensionCycleGC(AppTestCpythonExtensionBase):
 
-    def test_refcount(self):
-        import sys, gc
-        init = """
-        if (Py_IsInitialized()) {
-            PyObject* m;
+    def setup_method(self, func):
+        if self.runappdirect:
+            return
 
-            if (PyType_Ready(&CycleType) < 0)
-                return;
+        @unwrap_spec(methods='text')
+        def import_cycle_module(space, methods):
+            init = """
+            if (Py_IsInitialized()) {
+                PyObject* m;
+                if (PyType_Ready(&CycleType) < 0)
+                    return;
+                m = Py_InitModule("cycle", module_methods);
+                if (m == NULL)
+                    return;
+                Py_INCREF(&CycleType);
+                PyModule_AddObject(m, "Cycle", (PyObject *)&CycleType);
+            }
+            """
+            body = """
+            #include <Python.h>
+            #include "structmember.h"
+            typedef struct {
+                PyObject_HEAD
+                PyObject *next;
+                PyObject *val;
+            } Cycle;
+            static PyTypeObject CycleType;
+            static int Cycle_traverse(Cycle *self, visitproc visit, void *arg)
+            {
+                int vret;
+                if (self->next) {
+                    vret = visit(self->next, arg);
+                    if (vret != 0)
+                        return vret;
+                }
+                if (self->val) {
+                    vret = visit(self->val, arg);
+                    if (vret != 0)
+                        return vret;
+                }
+                return 0;
+            }
+            static int Cycle_clear(Cycle *self)
+            {
+                PyObject *tmp;
+                tmp = self->next;
+                self->next = NULL;
+                Py_XDECREF(tmp);
+                tmp = self->val;
+                self->val = NULL;
+                Py_XDECREF(tmp);
+                return 0;
+            }
+            static void Cycle_dealloc(Cycle* self)
+            {
+                Cycle_clear(self);
+                Py_TYPE(self)->tp_free((PyObject*)self);
+            }
+            static PyObject* Cycle_new(PyTypeObject *type, PyObject *args,
+                                       PyObject *kwds)
+            {
+                Cycle *self;
+                self = (Cycle *)type->tp_alloc(type, 0);
+                if (self != NULL) {
+                    self->next = PyString_FromString("");
+                    if (self->next == NULL) {
+                        Py_DECREF(self);
+                        return NULL;
+                    }
+                }
+                return (PyObject *)self;
+            }
+            static int Cycle_init(Cycle *self, PyObject *args, PyObject *kwds)
+            {
+                PyObject *next=NULL, *tmp;
+                static char *kwlist[] = {"next", NULL};
+                if (! PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist,
+                                                  &next))
+                    return -1;
+                if (next) {
+                    tmp = self->next;
+                    Py_INCREF(next);
+                    self->next = next;
+                    Py_XDECREF(tmp);
+                }
+                return 0;
+            }
+            static PyMemberDef Cycle_members[] = {
+                {"next", T_OBJECT_EX, offsetof(Cycle, next), 0, "next"},
+                {"val", T_OBJECT_EX, offsetof(Cycle, val), 0, "val"},
+                {NULL}  /* Sentinel */
+            };
+            static PyMethodDef Cycle_methods[] = {
+                {NULL}  /* Sentinel */
+            };
+            static PyTypeObject CycleType = {
+                PyVarObject_HEAD_INIT(NULL, 0)
+                "Cycle.Cycle",             /* tp_name */
+                sizeof(Cycle),             /* tp_basicsize */
+                0,                         /* tp_itemsize */
+                (destructor)Cycle_dealloc, /* tp_dealloc */
+                0,                         /* tp_print */
+                0,                         /* tp_getattr */
+                0,                         /* tp_setattr */
+                0,                         /* tp_compare */
+                0,                         /* tp_repr */
+                0,                         /* tp_as_number */
+                0,                         /* tp_as_sequence */
+                0,                         /* tp_as_mapping */
+                0,                         /* tp_hash */
+                0,                         /* tp_call */
+                0,                         /* tp_str */
+                0,                         /* tp_getattro */
+                0,                         /* tp_setattro */
+                0,                         /* tp_as_buffer */
+                Py_TPFLAGS_DEFAULT |
+                    Py_TPFLAGS_BASETYPE |
+                    Py_TPFLAGS_HAVE_GC,    /* tp_flags */
+                "Cycle objects",           /* tp_doc */
+                (traverseproc)Cycle_traverse,   /* tp_traverse */
+                (inquiry)Cycle_clear,           /* tp_clear */
+                0,                         /* tp_richcompare */
+                0,                         /* tp_weaklistoffset */
+                0,                         /* tp_iter */
+                0,                         /* tp_iternext */
+                Cycle_methods,             /* tp_methods */
+                Cycle_members,             /* tp_members */
+                0,                         /* tp_getset */
+                0,                         /* tp_base */
+                0,                         /* tp_dict */
+                0,                         /* tp_descr_get */
+                0,                         /* tp_descr_set */
+                0,                         /* tp_dictoffset */
+                (initproc)Cycle_init,      /* tp_init */
+                0,                         /* tp_alloc */
+                Cycle_new,                 /* tp_new */
+            };
+            """
+            w_result = self.sys_info.import_module("cycle", init,
+                                                   body + methods,
+                                                   None, None, False)
+            return w_result
 
-            m = Py_InitModule3("cycle", module_methods,
-                            "Example module that creates an extension type.");
+        self.imported_module_names = []
 
-            if (m == NULL)
-                return;
+        wrap = self.space.wrap
+        self.w_import_cycle_module = wrap(interp2app(import_cycle_module))
+        self.w_collect = wrap(interp2app(collect))
 
-            Py_INCREF(&CycleType);
-            PyModule_AddObject(m, "Cycle", (PyObject *)&CycleType);
-        }
-        """
-        body = """
-                #include <Python.h>
-                #include "structmember.h"
-
-                typedef struct {
-                    PyObject_HEAD
-                    PyObject *next;
-                    PyObject *val;
-                } Cycle;
-
-                static PyTypeObject CycleType;
-
-                static int
-                Cycle_traverse(Cycle *self, visitproc visit, void *arg)
-                {
-                    int vret;
-
-                    if (self->next) {
-                        vret = visit(self->next, arg);
-                        if (vret != 0)
-                            return vret;
-                    }
-                    if (self->val) {
-                        vret = visit(self->val, arg);
-                        if (vret != 0)
-                            return vret;
-                    }
-
-                    return 0;
-                }
-
-                static int
-                Cycle_clear(Cycle *self)
-                {
-                    PyObject *tmp;
-
-                    tmp = self->next;
-                    self->next = NULL;
-                    Py_XDECREF(tmp);
-
-                    tmp = self->val;
-                    self->val = NULL;
-                    Py_XDECREF(tmp);
-
-                    return 0;
-                }
-
-                static void
-                Cycle_dealloc(Cycle* self)
-                {
-                    Cycle_clear(self);
-                    Py_TYPE(self)->tp_free((PyObject*)self);
-                }
-
-                static PyObject *
-                Cycle_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
-                {
-                    Cycle *self;
-
-                    self = (Cycle *)type->tp_alloc(type, 0);
-                    if (self != NULL) {
-                        self->next = PyString_FromString("");
-                        if (self->next == NULL) {
-                            Py_DECREF(self);
-                            return NULL;
-                        }
-                    }
-
-                    return (PyObject *)self;
-                }
-
-                static int
-                Cycle_init(Cycle *self, PyObject *args, PyObject *kwds)
-                {
-                    PyObject *next=NULL, *tmp;
-
-                    static char *kwlist[] = {"next", NULL};
-
-                    if (! PyArg_ParseTupleAndKeywords(args, kwds, "|O", kwlist,
-                                                    &next))
-                        return -1;
-
-                    if (next) {
-                        tmp = self->next;
-                        Py_INCREF(next);
-                        self->next = next;
-                        Py_XDECREF(tmp);
-                    }
-
-                    return 0;
-                }
-
-                static Cycle *lastCreated;
-
-                static PyObject * Cycle_createCycle(Cycle *self, PyObject *val)
-                {
-                    Cycle *c;
-
-                    c = PyObject_GC_New(Cycle, &CycleType);
-                    if (c == NULL)
-                        return NULL;
-
-                    // set value
-                    Py_INCREF(val);
-                    c->val = val;
-
-                    // track by GC
-                    PyObject_GC_Track(c);
-
-                    // create cycle
-                    Py_INCREF(c);
-                    c->next = (PyObject *)c;
-
-
-                    // save c, but do no INCREF -> reference in lastCreated might become broken
-                    lastCreated = (Cycle *)c;
-
-                    // throw away reference to c
-                    Py_DECREF(c);
-
-                    // return None
-                    Py_INCREF(Py_None);
-                    return Py_None;
-                }
-
-                static PyObject * Cycle_breakCycle(Cycle *self)
-                {
-                    Cycle *tmp;
-
-                    // set next to None
-                    tmp = (Cycle *)lastCreated->next;
-                    Py_INCREF(Py_None);
-                    lastCreated->next = Py_None;
-                    Py_XDECREF(tmp);
-
-                    // return None
-                    Py_INCREF(Py_None);
-                    return Py_None;
-                }
-
-                static PyMemberDef Cycle_members[] = {
-                    {"next", T_OBJECT_EX, offsetof(Cycle, next), 0,
-                    "next"},
-                    {"val", T_OBJECT_EX, offsetof(Cycle, val), 0,
-                    "val"},
-                    {NULL}  /* Sentinel */
-                };
-
-                static PyMethodDef Cycle_methods[] = {
-                    {NULL}  /* Sentinel */
-                };
-
-                static PyTypeObject CycleType = {
-                    PyVarObject_HEAD_INIT(NULL, 0)
-                    "Cycle.Cycle",             /* tp_name */
-                    sizeof(Cycle),             /* tp_basicsize */
-                    0,                         /* tp_itemsize */
-                    (destructor)Cycle_dealloc, /* tp_dealloc */
-                    0,                         /* tp_print */
-                    0,                         /* tp_getattr */
-                    0,                         /* tp_setattr */
-                    0,                         /* tp_compare */
-                    0,                         /* tp_repr */
-                    0,                         /* tp_as_number */
-                    0,                         /* tp_as_sequence */
-                    0,                         /* tp_as_mapping */
-                    0,                         /* tp_hash */
-                    0,                         /* tp_call */
-                    0,                         /* tp_str */
-                    0,                         /* tp_getattro */
-                    0,                         /* tp_setattro */
-                    0,                         /* tp_as_buffer */
-                    Py_TPFLAGS_DEFAULT |
-                        Py_TPFLAGS_BASETYPE |
-                        Py_TPFLAGS_HAVE_GC,    /* tp_flags */
-                    "Cycle objects",           /* tp_doc */
-                    (traverseproc)Cycle_traverse,   /* tp_traverse */
-                    (inquiry)Cycle_clear,           /* tp_clear */
-                    0,                         /* tp_richcompare */
-                    0,                         /* tp_weaklistoffset */
-                    0,                         /* tp_iter */
-                    0,                         /* tp_iternext */
-                    Cycle_methods,             /* tp_methods */
-                    Cycle_members,             /* tp_members */
-                    0,                         /* tp_getset */
-                    0,                         /* tp_base */
-                    0,                         /* tp_dict */
-                    0,                         /* tp_descr_get */
-                    0,                         /* tp_descr_set */
-                    0,                         /* tp_dictoffset */
-                    (initproc)Cycle_init,      /* tp_init */
-                    0,                         /* tp_alloc */
-                    Cycle_new,                 /* tp_new */
-                };
-
-                static PyMethodDef module_methods[] = {
-                    {"breakCycle", (PyCFunction)Cycle_breakCycle, METH_NOARGS, "break cycle"},
-                    {"createCycle", (PyCFunction)Cycle_createCycle, METH_OLDARGS, "create a special cycle"},
-                    {NULL}  /* Sentinel */
-                };
-        """
-        cycle = self.import_module(name='cycle', init=init, body=body)
+    def test_free_self_reference_cycle_child_pypyobj(self):
+        cycle = self.import_cycle_module("""
+            static Cycle *c;
+            static PyObject * Cycle_cc(Cycle *self, PyObject *val)
+            {
+                c = PyObject_GC_New(Cycle, &CycleType);
+                if (c == NULL)
+                    return NULL;
+                Py_INCREF(val);
+                c->val = val;                // set value
+                PyObject_GC_Track(c);
+                Py_INCREF(c);
+                c->next = (PyObject *)c;     // create self reference
+                Py_INCREF(Py_None);
+                return Py_None;
+            }
+            static PyObject * Cycle_cd(Cycle *self)
+            {
+                Py_DECREF(c);                // throw cycle away
+                Py_INCREF(Py_None);
+                return Py_None;
+            }
+            static PyMethodDef module_methods[] = {
+                {"createCycle", (PyCFunction)Cycle_cc, METH_OLDARGS, ""},
+                {"discardCycle", (PyCFunction)Cycle_cd, METH_NOARGS, ""},
+                {NULL}  /* Sentinel */
+            };
+            """)
 
         class Example(object):
-            def __init__(self):
-                self.next = None
+            del_called = -1
+
+            def __init__(self, val):
+                self.val = val
+                Example.del_called = 0
 
             def __del__(self):
-                print("free")
+                Example.del_called = self.val
 
-        cycle.createCycle(Example())
+        # don't keep any reference in pypy
+        cycle.createCycle(Example(42))
+        self.collect()
+        assert Example.del_called == 0
+        cycle.discardCycle()
+        self.collect()
+        assert Example.del_called == 42
 
-        gc.collect()
+        # keep a temporary reference in pypy
+        e = Example(43)
+        cycle.createCycle(e)
+        cycle.discardCycle()
+        self.collect()
+        assert Example.del_called == 0
+        e = None
+        self.collect()
+        assert Example.del_called == 43
 
-        # check if object has been freed
+        # keep a reference in pypy, free afterwards
+        e = Example(44)
+        cycle.createCycle(e)
+        self.collect()
+        assert Example.del_called == 0
+        e = None
+        self.collect()
+        assert Example.del_called == 0
+        cycle.discardCycle()
+        self.collect()
+        assert Example.del_called == 44
 
-        assert False
+    def test_free_self_reference_cycle_parent_pypyobj(self):
+        # create and return a second object which references the cycle, because
+        # otherwise we will end up with a cycle that spans across cpy/pypy,
+        # which we don't want to test here
+        cycle = self.import_cycle_module("""
+            static PyObject * Cycle_cc(Cycle *self, PyObject *val)
+            {
+                Cycle *c = PyObject_GC_New(Cycle, &CycleType);
+                if (c == NULL)
+                    return NULL;
+                Cycle *c2 = PyObject_GC_New(Cycle, &CycleType);
+                if (c2 == NULL)
+                    return NULL;
+                Py_INCREF(val);
+                c2->val = val;                // set value
+                PyObject_GC_Track(c);
+                PyObject_GC_Track(c2);
+                Py_INCREF(c2);
+                c2->next = (PyObject *)c2;    // create self reference
+                c->next = (PyObject *)c2;
+                return (PyObject *)c;         // return other object
+            }
+            static PyMethodDef module_methods[] = {
+                {"createCycle", (PyCFunction)Cycle_cc, METH_OLDARGS, ""},
+                {NULL}  /* Sentinel */
+            };
+            """)
+
+        class Example(object):
+            del_called = -1
+
+            def __init__(self, val):
+                self.val = val
+                Example.del_called = 0
+
+            def __del__(self):
+                Example.del_called = self.val
+
+        c = cycle.createCycle(Example(42))
+        self.collect()
+        assert Example.del_called == 0
+        c = None
+        self.collect()
+        assert Example.del_called == 42
+
+    def test_free_simple_cycle_child_pypyobj(self):
+        cycle = self.import_cycle_module("""
+            static Cycle *c;
+            static PyObject * Cycle_cc(Cycle *self, PyObject *val)
+            {
+                c = PyObject_GC_New(Cycle, &CycleType);
+                if (c == NULL)
+                    return NULL;
+                Cycle *c2 = PyObject_GC_New(Cycle, &CycleType);
+                if (c2 == NULL)
+                    return NULL;
+                Py_INCREF(val);
+                c->val = val;                // set value
+                PyObject_GC_Track(c);
+                PyObject_GC_Track(c2);
+                c->next = (PyObject *)c2;
+                Py_INCREF(c);
+                c2->next = (PyObject *)c;    // simple cycle across two objects
+                Py_INCREF(Py_None);
+                return Py_None;
+            }
+            static PyObject * Cycle_cd(Cycle *self)
+            {
+                Py_DECREF(c);                // throw cycle away
+                Py_INCREF(Py_None);
+                return Py_None;
+            }
+            static PyMethodDef module_methods[] = {
+                {"createCycle", (PyCFunction)Cycle_cc, METH_OLDARGS, ""},
+                {"discardCycle", (PyCFunction)Cycle_cd, METH_NOARGS, ""},
+                {NULL}  /* Sentinel */
+            };
+            """)
+
+        class Example(object):
+            del_called = -1
+
+            def __init__(self, val):
+                self.val = val
+                Example.del_called = 0
+
+            def __del__(self):
+                Example.del_called = self.val
+
+        # don't keep any reference in pypy
+        cycle.createCycle(Example(42))
+        self.collect()
+        cycle.discardCycle()
+        assert Example.del_called == 0
+        self.collect()
+        assert Example.del_called == 42
+
+        # keep a temporary reference in pypy
+        e = Example(43)
+        cycle.createCycle(e)
+        cycle.discardCycle()
+        self.collect()
+        assert Example.del_called == 0
+        e = None
+        self.collect()
+        assert Example.del_called == 43
+
+        # keep a reference in pypy, free afterwards
+        e = Example(44)
+        cycle.createCycle(e)
+        self.collect()
+        assert Example.del_called == 0
+        e = None
+        self.collect()
+        assert Example.del_called == 0
+        cycle.discardCycle()
+        self.collect()
+        assert Example.del_called == 44
+
+    def test_free_complex_cycle_child_pypyobj(self):
+        cycle = self.import_cycle_module("""
+            static PyObject * Cycle_cc(Cycle *self, PyObject *val)
+            {
+                Cycle *c = PyObject_GC_New(Cycle, &CycleType);
+                if (c == NULL)
+                    return NULL;
+                Cycle *c2 = PyObject_GC_New(Cycle, &CycleType);
+                if (c2 == NULL)
+                    return NULL;
+                Cycle *c3 = PyObject_GC_New(Cycle, &CycleType);
+                if (c3 == NULL)
+                    return NULL;
+                Py_INCREF(val);
+                c->val = val;                // set value
+                Py_INCREF(val);
+                c3->val = val;                // set value
+                PyObject_GC_Track(c);
+                PyObject_GC_Track(c2);
+                PyObject_GC_Track(c3);
+                Py_INCREF(c2);
+                c->next = (PyObject *)c2;
+                Py_INCREF(c);
+                c2->next = (PyObject *)c;    // inner cycle
+                Py_INCREF(c3);
+                c2->val = (PyObject *)c3;
+                Py_INCREF(c);
+                c3->next = (PyObject *)c;     // outer cycle
+                Py_DECREF(c);
+                Py_DECREF(c2);
+                Py_DECREF(c3);               // throw all objects away
+                Py_INCREF(Py_None);
+                return Py_None;
+            }
+            static PyMethodDef module_methods[] = {
+                {"createCycle", (PyCFunction)Cycle_cc, METH_OLDARGS, ""},
+                {NULL}  /* Sentinel */
+            };
+            """)
+
+        class Example(object):
+            del_called = -1
+
+            def __init__(self, val):
+                self.val = val
+                Example.del_called = 0
+
+            def __del__(self):
+                Example.del_called = self.val
+
+        # don't keep any reference in pypy
+        cycle.createCycle(Example(42))
+        assert Example.del_called == 0
+        self.collect()
+        assert Example.del_called == 42
+
+        # keep a temporary reference in pypy
+        e = Example(43)
+        cycle.createCycle(e)
+        e = None
+        assert Example.del_called == 0
+        self.collect()
+        assert Example.del_called == 43
+
+        # keep a reference in pypy, free afterwards
+        e = Example(44)
+        cycle.createCycle(e)
+        self.collect()
+        assert Example.del_called == 0
+        e = None
+        self.collect()
+        assert Example.del_called == 44
diff --git a/rpython/rlib/rawrefcount.py b/rpython/rlib/rawrefcount.py
--- a/rpython/rlib/rawrefcount.py
+++ b/rpython/rlib/rawrefcount.py
@@ -23,6 +23,26 @@
     _adr2pypy.append(p)
     return res
 
+_trial_deletion_phase = 0
+_visited = []
+_marked = []
+
+def set_trialdeletion_phase(value):
+    _trial_deletion_phase = value
+def get_trialdeletion_phase():
+    return _trial_deletion_phase
+def add_visited(obj):
+    _visited.append(obj)
+def get_visited():
+    return _visited
+def clear_visited():
+    del _visited[:]
+def add_marked(obj):
+    _marked.append(obj)
+def get_marked():
+    return marked
+def clear_marked():
+    del _marked[:]
 
 @not_rpython
 def init(dealloc_trigger_callback=None):
diff --git a/rpython/rlib/rgc.py b/rpython/rlib/rgc.py
--- a/rpython/rlib/rgc.py
+++ b/rpython/rlib/rgc.py
@@ -522,8 +522,8 @@
         translator = hop.rtyper.annotator.translator
         fq = hop.args_s[0].const
         graph = translator._graphof(fq.finalizer_trigger.im_func)
-        InstanceRepr.check_graph_of_del_does_not_call_too_much(hop.rtyper,
-                                                               graph)
+        #InstanceRepr.check_graph_of_del_does_not_call_too_much(hop.rtyper,
+        #                                                       graph)
         hop.exception_cannot_occur()
         return hop.inputconst(lltype.Signed, hop.s_result.const)
 
diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py
--- a/rpython/rtyper/rclass.py
+++ b/rpython/rtyper/rclass.py
@@ -585,8 +585,8 @@
                 assert len(s_func.descriptions) == 1
                 funcdesc, = s_func.descriptions
                 graph = funcdesc.getuniquegraph()
-                self.check_graph_of_del_does_not_call_too_much(self.rtyper,
-                                                               graph)
+                #self.check_graph_of_del_does_not_call_too_much(self.rtyper,
+                #                                               graph)
                 FUNCTYPE = FuncType([Ptr(source_repr.object_type)], Void)
                 destrptr = functionptr(FUNCTYPE, graph.name,
                                        graph=graph,

From pypy.commits at gmail.com  Sat Jun 24 07:35:08 2017
From: pypy.commits at gmail.com (arigo)
Date: Sat, 24 Jun 2017 04:35:08 -0700 (PDT)
Subject: [pypy-commit] pypy default: Finally understood why these two tests
 fails on 32-bit. Skip them as
Message-ID: <594e4e6c.0ca8df0a.5f562.da56@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91644:b9047ff141b7
Date: 2017-06-24 13:34 +0200
http://bitbucket.org/pypy/pypy/changeset/b9047ff141b7/

Log:	Finally understood why these two tests fails on 32-bit. Skip them as
	"not really worth worrying about".

diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -753,6 +753,9 @@
 
 @rposix_requires('posix_fadvise')
 def test_posix_fadvise():
+    if sys.maxint <= 2**32:
+        py.test.skip("ll2ctypes run of posix_fadvise() on 32-bit "
+                     "gets confused by the size of OFF_T")
     fname = str(udir.join('test_os_posix_fadvise'))
     fd = os.open(fname, os.O_CREAT | os.O_RDWR)
     try:
@@ -769,6 +772,9 @@
 
 @rposix_requires('posix_fallocate')
 def test_posix_fallocate():
+    if sys.maxint <= 2**32:
+        py.test.skip("ll2ctypes run of posix_fallocate() on 32-bit "
+                     "gets confused by the size of OFF_T")
     fname = str(udir.join('os_test.txt'))
     fd = os.open(fname, os.O_WRONLY | os.O_CREAT, 0777)
     try:

From pypy.commits at gmail.com  Sun Jun 25 03:44:10 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Jun 2017 00:44:10 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Mention that this change was backported
 to cryptography
Message-ID: <594f69ca.45b4df0a.a7a70.54ae@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91645:f60ae489d83a
Date: 2017-06-25 09:43 +0200
http://bitbucket.org/pypy/pypy/changeset/f60ae489d83a/

Log:	Mention that this change was backported to cryptography

diff --git a/lib_pypy/_cffi_ssl/README.md b/lib_pypy/_cffi_ssl/README.md
--- a/lib_pypy/_cffi_ssl/README.md
+++ b/lib_pypy/_cffi_ssl/README.md
@@ -7,7 +7,7 @@
 
 NOTE: currently, we have changed ``_cffi_src/openssl/callbacks.py`` to
 not rely on the CPython C API, and ``_cffi_src/utils.py`` for issue #2575
-(29c9a89359e4).
+(29c9a89359e4).  (The first change is now backported.)
 
 # Tests?
 

From pypy.commits at gmail.com  Sun Jun 25 05:43:47 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Jun 2017 02:43:47 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2579
Message-ID: <594f85d3.438b1c0a.f04db.3ab6@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91646:fd92a8bc06b6
Date: 2017-06-25 11:43 +0200
http://bitbucket.org/pypy/pypy/changeset/fd92a8bc06b6/

Log:	Issue #2579

	First approach to fix a "leak" in _lzma.py

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -10,6 +10,7 @@
 import weakref
 import sys
 import io
+import __pypy__
 
 from _lzma_cffi import ffi, lib as m
 
@@ -688,6 +689,16 @@
         self.lock = threading.Lock()
         self.flushed = 0
         self.lzs = _new_lzma_stream()
+        # Issue #2579:
+        # Setting up the stream for encoding takes around 17MB of RAM
+        # on my Linux 64 system.  That's potentially too much to sanely
+        # leave it to the GC: in case we're compressing a large number
+        # of small files, the following line puts a big pressure on the
+        # major collections.  Still better than without it, where it
+        # would allocate huge amount of RAMs before doing any collection.
+        # Ideally we should do something more clever like reusing old
+        # streams after we're finished with them.
+        __pypy__.add_memory_pressure(1024*1024*17)
         if format == FORMAT_XZ:
             if filters is None:
                 if check == -1:

From pypy.commits at gmail.com  Sun Jun 25 06:02:42 2017
From: pypy.commits at gmail.com (arigo)
Date: Sun, 25 Jun 2017 03:02:42 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: More GC tweaks. Trying out to call
 add_memory_pressure() with a
Message-ID: <594f8a42.8f0c1c0a.e2837.c9f3@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: py3.5
Changeset: r91647:45f61802dc1c
Date: 2017-06-25 12:02 +0200
http://bitbucket.org/pypy/pypy/changeset/45f61802dc1c/

Log:	More GC tweaks. Trying out to call add_memory_pressure() with a
	negative number. I *think* it should always give what we want.

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -64,6 +64,10 @@
     m._pylzma_stream_init(ret)
     return ffi.gc(ret, m.lzma_end)
 
+def _release_lzma_stream(st):
+    ffi.gc(st, None)
+    m.lzma_end(st)
+
 def add_constant(c):
     globals()[c] = getattr(m, 'LZMA_' + c)
 
@@ -649,6 +653,16 @@
         raise TypeError("cannot serialize '%s' object" %
                         self.__class__.__name__)
 
+
+# Issue #2579: Setting up the stream for encoding takes around 17MB of
+# RAM on my Linux 64 system.  So we call add_memory_pressure(17MB) when
+# we create the stream.  In flush(), we actively free the stream even
+# though we could just leave it to the GC (but 17MB is too much for
+# doing that sanely); at this point we call add_memory_pressure(-17MB)
+# to cancel the original increase.
+COMPRESSION_STREAM_SIZE = 1024*1024*17
+
+
 class LZMACompressor(object):
     """
     LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None)
@@ -689,16 +703,7 @@
         self.lock = threading.Lock()
         self.flushed = 0
         self.lzs = _new_lzma_stream()
-        # Issue #2579:
-        # Setting up the stream for encoding takes around 17MB of RAM
-        # on my Linux 64 system.  That's potentially too much to sanely
-        # leave it to the GC: in case we're compressing a large number
-        # of small files, the following line puts a big pressure on the
-        # major collections.  Still better than without it, where it
-        # would allocate huge amount of RAMs before doing any collection.
-        # Ideally we should do something more clever like reusing old
-        # streams after we're finished with them.
-        __pypy__.add_memory_pressure(1024*1024*17)
+        __pypy__.add_memory_pressure(COMPRESSION_STREAM_SIZE)
         if format == FORMAT_XZ:
             if filters is None:
                 if check == -1:
@@ -782,7 +787,10 @@
             if self.flushed:
                 raise ValueError("Repeated...")
             self.flushed = 1
-            return self._compress(b'', action=m.LZMA_FINISH)
+            result = self._compress(b'', action=m.LZMA_FINISH)
+            __pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE)
+            _release_lzma_stream(self.lzs)
+        return result
 
     def __getstate__(self):
         raise TypeError("cannot serialize '%s' object" %

From pypy.commits at gmail.com  Sun Jun 25 15:25:44 2017
From: pypy.commits at gmail.com (rlamy)
Date: Sun, 25 Jun 2017 12:25:44 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Add failing extra_test (inspired by
 CPython's deccheck.py)
Message-ID: <59500e38.d0141c0a.33b09.b93a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91648:b7ddeba4af0d
Date: 2017-06-25 20:25 +0100
http://bitbucket.org/pypy/pypy/changeset/b7ddeba4af0d/

Log:	Add failing extra_test (inspired by CPython's deccheck.py)

diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
--- a/extra_tests/test_decimal.py
+++ b/extra_tests/test_decimal.py
@@ -1,4 +1,5 @@
 import pytest
+from hypothesis import given, strategies as st
 
 import pickle
 import sys
@@ -14,6 +15,27 @@
 def module(request):
     yield request.param
 
+# Translate symbols.
+CondMap = {
+        C.Clamped:             P.Clamped,
+        C.ConversionSyntax:    P.ConversionSyntax,
+        C.DivisionByZero:      P.DivisionByZero,
+        C.DivisionImpossible:  P.InvalidOperation,
+        C.DivisionUndefined:   P.DivisionUndefined,
+        C.Inexact:             P.Inexact,
+        C.InvalidContext:      P.InvalidContext,
+        C.InvalidOperation:    P.InvalidOperation,
+        C.Overflow:            P.Overflow,
+        C.Rounded:             P.Rounded,
+        C.Subnormal:           P.Subnormal,
+        C.Underflow:           P.Underflow,
+        C.FloatOperation:      P.FloatOperation,
+}
+
+def check_same_flags(flags_C, flags_P):
+    for signal in flags_C:
+        assert flags_C[signal] == flags_P[CondMap[signal]]
+
 
 def test_C():
     sys.modules["decimal"] = C
@@ -68,3 +90,22 @@
 
 def test_compare_total_mag(module):
     assert module.Decimal(1).compare_total_mag(-2) == -1
+
+ at given(st.decimals(), st.decimals())
+def test_lt(d1, d2):
+    ctx_C = C.getcontext()
+    ctx_P = P.getcontext()
+    ctx_C.clear_flags()
+    ctx_P.clear_flags()
+    d1_P = P.Decimal(str(d1))
+    d2_P = P.Decimal(str(d2))
+    try:
+        res_C = d1 < d2
+    except Exception as e:
+        res_C = str(type(e))
+    try:
+        res_P = d1_P < d2_P
+    except Exception as e:
+        res_P = str(type(e))
+    assert res_C == res_P
+    check_same_flags(ctx_C.flags, ctx_P.flags)

From pypy.commits at gmail.com  Mon Jun 26 13:27:58 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Jun 2017 10:27:58 -0700 (PDT)
Subject: [pypy-commit] pypy default: Issue #2595: copies the test, and fix
Message-ID: <5951441e.5e361c0a.5e748.24c8@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91649:dc929547a198
Date: 2017-06-26 19:27 +0200
http://bitbucket.org/pypy/pypy/changeset/dc929547a198/

Log:	Issue #2595: copies the test, and fix

diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -268,12 +268,22 @@
         assert abs(d) == 1
         source = getcurrent()
         source.tempval = arg
-        if d > 0:
-            cando = self.balance < 0
-            dir = d
-        else:
-            cando = self.balance > 0
-            dir = 0
+        while True:
+            if d > 0:
+                cando = self.balance < 0
+                dir = d
+            else:
+                cando = self.balance > 0
+                dir = 0
+
+            if cando and not self.queue[0].alive:
+                # issue #2595: the tasklet was killed while waiting.
+                # drop that tasklet from consideration and try again.
+                self.balance += d
+                self.queue.popleft()
+            else:
+                # normal path
+                break
 
         if _channel_callback is not None:
             _channel_callback(self, source, dir, not cando)
diff --git a/pypy/module/test_lib_pypy/test_stackless.py b/pypy/module/test_lib_pypy/test_stackless.py
--- a/pypy/module/test_lib_pypy/test_stackless.py
+++ b/pypy/module/test_lib_pypy/test_stackless.py
@@ -600,4 +600,23 @@
 
         stackless.run()
 
-
+    def test_kill_tasklet_waiting_for_channel(self):
+        # issue #2595
+        c = stackless.channel()
+        def sender():
+            c.send(1)
+        def receiver():
+            v = c.receive()
+        def killer(tl):
+            tl.kill()
+        def main():
+            trk = stackless.tasklet(receiver)()
+            stackless.schedule()
+            killer(trk)
+            stackless.schedule()
+            stackless.tasklet(sender)()
+            stackless.schedule()
+            stackless.tasklet(receiver)()
+            stackless.schedule()
+        stackless.tasklet(main)()
+        stackless.run()

From pypy.commits at gmail.com  Mon Jun 26 13:39:03 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 26 Jun 2017 10:39:03 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: fix test
Message-ID: <595146b7.8f0c1c0a.e8ee2.0c6a@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91651:9e54509d2d8f
Date: 2017-06-26 18:38 +0100
http://bitbucket.org/pypy/pypy/changeset/9e54509d2d8f/

Log:	fix test

diff --git a/extra_tests/test_decimal.py b/extra_tests/test_decimal.py
--- a/extra_tests/test_decimal.py
+++ b/extra_tests/test_decimal.py
@@ -1,5 +1,5 @@
 import pytest
-from hypothesis import given, strategies as st
+from hypothesis import example, settings, given, strategies as st
 
 import pickle
 import sys
@@ -91,21 +91,32 @@
 def test_compare_total_mag(module):
     assert module.Decimal(1).compare_total_mag(-2) == -1
 
- at given(st.decimals(), st.decimals())
+def convert_arg(module, arg):
+    if isinstance(arg, module.Decimal):
+        return arg
+    elif type(arg).__name__ == 'Decimal':
+        return module.Decimal(str(arg))
+    else:
+        return arg
+
+from fractions import Fraction
+from decimal import Decimal
+
+ at given(st.decimals(), st.decimals() | st.fractions())
 def test_lt(d1, d2):
-    ctx_C = C.getcontext()
-    ctx_P = P.getcontext()
-    ctx_C.clear_flags()
-    ctx_P.clear_flags()
-    d1_P = P.Decimal(str(d1))
-    d2_P = P.Decimal(str(d2))
-    try:
-        res_C = d1 < d2
-    except Exception as e:
-        res_C = str(type(e))
-    try:
-        res_P = d1_P < d2_P
-    except Exception as e:
-        res_P = str(type(e))
+    with C.localcontext(C.ExtendedContext) as ctx_C:
+        d1_C = convert_arg(C, d1)
+        d2_C = convert_arg(C, d2)
+        try:
+            res_C = d1_C < d2_C
+        except Exception as e:
+            res_C = str(type(e))
+    with P.localcontext(P.ExtendedContext) as ctx_P:
+        d1_P = convert_arg(P, d1)
+        d2_P = convert_arg(P, d2)
+        try:
+            res_P = d1_P < d2_P
+        except Exception as e:
+            res_P = str(type(e))
     assert res_C == res_P
     check_same_flags(ctx_C.flags, ctx_P.flags)

From pypy.commits at gmail.com  Mon Jun 26 13:56:39 2017
From: pypy.commits at gmail.com (rlamy)
Date: Mon, 26 Jun 2017 10:56:39 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix issue #2583
Message-ID: <59514ad7.898c1c0a.49545.1f3d@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91652:333ccb80fc0e
Date: 2017-06-26 18:56 +0100
http://bitbucket.org/pypy/pypy/changeset/333ccb80fc0e/

Log:	Fix issue #2583

diff --git a/lib_pypy/_decimal.py b/lib_pypy/_decimal.py
--- a/lib_pypy/_decimal.py
+++ b/lib_pypy/_decimal.py
@@ -489,13 +489,16 @@
                     vv.exp = 0
                     multiplied = Decimal._new_empty()
                     denom = Decimal(other.denominator)
-                    with _CatchStatus(context) as (ctx, status_ptr):
-                        _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd,
-                                        ctx, status_ptr)
-                    multiplied._mpd.exp += exp  # XXX probably a bug
-                                                # in _decimal.c
+                    maxctx = _ffi.new("struct mpd_context_t*")
+                    _mpdec.mpd_maxcontext(maxctx)
+                    status_ptr = _ffi.new("uint32_t*")
+                    _mpdec.mpd_qmul(multiplied._mpd, vv, denom._mpd,
+                                    maxctx, status_ptr)
+                    multiplied._mpd.exp = exp
                 finally:
                     _mpdec.mpd_del(vv)
+                if status_ptr[0] != 0:
+                    raise ValueError("exact conversion for comparison failed")
 
                 return multiplied, numerator
             else:

From pypy.commits at gmail.com  Mon Jun 26 13:33:49 2017
From: pypy.commits at gmail.com (arigo)
Date: Mon, 26 Jun 2017 10:33:49 -0700 (PDT)
Subject: [pypy-commit] pypy default: oops, fix for dc929547a198
Message-ID: <5951457d.cb7e1c0a.15206.0d81@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: 
Changeset: r91650:29f3c769e610
Date: 2017-06-26 19:33 +0200
http://bitbucket.org/pypy/pypy/changeset/29f3c769e610/

Log:	oops, fix for dc929547a198

diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -276,7 +276,7 @@
                 cando = self.balance > 0
                 dir = 0
 
-            if cando and not self.queue[0].alive:
+            if cando and self.queue[0]._tasklet_killed:
                 # issue #2595: the tasklet was killed while waiting.
                 # drop that tasklet from consideration and try again.
                 self.balance += d
@@ -358,6 +358,8 @@
     module.
     """
     tempval = None
+    _tasklet_killed = False
+
     def __new__(cls, func=None, label=''):
         res = coroutine.__new__(cls)
         res.label = label
@@ -405,6 +407,7 @@
         If the exception passes the toplevel frame of the tasklet,
         the tasklet will silently die.
         """
+        self._tasklet_killed = True
         if not self.is_zombie:
             # Killing the tasklet by throwing TaskletExit exception.
             coroutine.kill(self)

From pypy.commits at gmail.com  Tue Jun 27 14:43:51 2017
From: pypy.commits at gmail.com (rlamy)
Date: Tue, 27 Jun 2017 11:43:51 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix #2578, but (I guess) only on POSIX
Message-ID: <5952a767.49891c0a.c882b.43d3@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91653:579fe19f0246
Date: 2017-06-27 19:43 +0100
http://bitbucket.org/pypy/pypy/changeset/579fe19f0246/

Log:	Fix #2578, but (I guess) only on POSIX

diff --git a/lib_pypy/_cffi_ssl/_stdssl/error.py b/lib_pypy/_cffi_ssl/_stdssl/error.py
--- a/lib_pypy/_cffi_ssl/_stdssl/error.py
+++ b/lib_pypy/_cffi_ssl/_stdssl/error.py
@@ -1,4 +1,5 @@
 import sys
+import os
 import traceback
 from _pypy_openssl import ffi
 from _pypy_openssl import lib
@@ -100,18 +101,17 @@
             errval = SSL_ERROR_WANT_CONNECT
         elif err == SSL_ERROR_SYSCALL:
             if e == 0:
-                if ret == 0 or obj.socket is not None:
+                if ret == 0 or obj.socket is None:
                     errtype = SSLEOFError
                     errstr = "EOF occurred in violation of protocol"
                     errval = SSL_ERROR_EOF
                 elif ret == -1 and obj.socket is not None:
                     # the underlying BIO reported an I/0 error
                     lib.ERR_clear_error()
-                    s = obj.get_socket_or_None()
-                    s.errorhandler()
-                    assert 0, "must not get here"
-                    #errno = ffi.errno
-                    #return IOError(errno)
+                    # s = obj.get_socket_or_None()
+                    # XXX: Windows?
+                    errno = ffi.errno
+                    return OSError(errno, os.strerror(errno))
                 else:
                     errtype = SSLSyscallError
                     errstr = "Some I/O error occurred"

From pypy.commits at gmail.com  Tue Jun 27 17:38:56 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 27 Jun 2017 14:38:56 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: As os.link() implementation on Windows
Message-ID: <5952d070.d2181c0a.3fa7a.22b9@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91654:16f97608b715
Date: 2017-05-06 11:46 +0200
http://bitbucket.org/pypy/pypy/changeset/16f97608b715/

Log:	As os.link() implementation on Windows

diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py
--- a/pypy/module/posix/__init__.py
+++ b/pypy/module/posix/__init__.py
@@ -69,6 +69,7 @@
         'chmod': 'interp_posix.chmod',
         'rename': 'interp_posix.rename',
         'replace': 'interp_posix.replace',
+        'link': 'interp_posix.link',
         'umask': 'interp_posix.umask',
         '_exit': 'interp_posix._exit',
         'utime': 'interp_posix.utime',
@@ -109,8 +110,6 @@
         interpleveldefs['killpg'] = 'interp_posix.killpg'
     if hasattr(os, 'getpid'):
         interpleveldefs['getpid'] = 'interp_posix.getpid'
-    if hasattr(os, 'link'):
-        interpleveldefs['link'] = 'interp_posix.link'
     if hasattr(os, 'symlink'):
         interpleveldefs['symlink'] = 'interp_posix.symlink'
     if hasattr(os, 'readlink'):
diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1296,9 +1296,17 @@
 @replace_os_function('link')
 @specialize.argtype(0, 1)
 def link(oldpath, newpath):
-    oldpath = _as_bytes0(oldpath)
-    newpath = _as_bytes0(newpath)
-    handle_posix_error('link', c_link(oldpath, newpath))
+    if not _WIN32:
+        oldpath = _as_bytes0(oldpath)
+        newpath = _as_bytes0(newpath)
+        handle_posix_error('link', c_link(oldpath, newpath))
+    else:
+        traits = _preferred_traits(path1)
+        win32traits = make_win32_traits(traits)
+        oldpath = traits.as_str0(oldpath)
+        newpath = traits.as_str0(newpath)
+        if not win32traits.CreateHardLink(newpath, oldpath, None):
+            raise rwin32.lastSavedWindowsError()
 
 @replace_os_function('symlink')
 @specialize.argtype(0, 1)
diff --git a/rpython/rlib/rwin32file.py b/rpython/rlib/rwin32file.py
--- a/rpython/rlib/rwin32file.py
+++ b/rpython/rlib/rwin32file.py
@@ -234,6 +234,12 @@
             rwin32.BOOL,
             save_err=rffi.RFFI_SAVE_LASTERROR)
 
+        CreateHardLink = external(
+            'CreateHardLink' + suffix,
+            [traits.CCHARP, traits.CCHARP, rwin32.LPSECURITY_ATTRIBUTES],
+            rwin32.BOOL,
+            save_err=rffi.RFFI_SAVE_LASTERROR)
+
     return Win32Traits
 
 def make_longlong(high, low):

From pypy.commits at gmail.com  Tue Jun 27 17:38:58 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 27 Jun 2017 14:38:58 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Issue #2594: Fix truncated errors
 messages in _lzma.py
Message-ID: <5952d072.153f1c0a.ad76d.c078@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91655:38e0caea905c
Date: 2017-06-27 23:36 +0200
http://bitbucket.org/pypy/pypy/changeset/38e0caea905c/

Log:	Issue #2594: Fix truncated errors messages in _lzma.py

	Full messages were found in CPython _lzmamodule.c

diff --git a/lib_pypy/_lzma.py b/lib_pypy/_lzma.py
--- a/lib_pypy/_lzma.py
+++ b/lib_pypy/_lzma.py
@@ -153,39 +153,39 @@
 def parse_filter_spec_lzma(id, preset=m.LZMA_PRESET_DEFAULT, **kwargs):
     ret = ffi.new('lzma_options_lzma*')
     if m.lzma_lzma_preset(ret, preset):
-        raise LZMAError("Invalid...")
+        raise LZMAError("Invalid compression preset: %s" % preset)
     for arg, val in kwargs.items():
         if arg in ('dict_size', 'lc', 'lp', 'pb', 'nice_len', 'depth'):
             setattr(ret, arg, val)
         elif arg in ('mf', 'mode'):
             setattr(ret, arg, int(val))
         else:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for LZMA filter")
     return ret
 
 def parse_filter_spec(spec):
     if not isinstance(spec, collections.Mapping):
-        raise TypeError("Filter...")
+        raise TypeError("Filter specifier must be a dict or dict-like object")
     ret = ffi.new('lzma_filter*')
     try:
         ret.id = spec['id']
     except KeyError:
-        raise ValueError("Filter...")
+        raise ValueError("Filter specifier must have an \"id\" entry")
     if ret.id in (m.LZMA_FILTER_LZMA1, m.LZMA_FILTER_LZMA2):
         try:
             options = parse_filter_spec_lzma(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for LZMA filter")
     elif ret.id == m.LZMA_FILTER_DELTA:
         try:
             options = parse_filter_spec_delta(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for delta filter")
     elif ret.id in BCJ_FILTERS:
         try:
             options = parse_filter_spec_bcj(**spec)
         except TypeError:
-            raise ValueError("Invalid...")
+            raise ValueError("Invalid filter specifier for BCJ filter")
     else:
         raise ValueError("Invalid %d" % (ret.id,))
 
@@ -209,7 +209,9 @@
 
 def parse_filter_chain_spec(filterspecs):
     if len(filterspecs) > m.LZMA_FILTERS_MAX:
-        raise ValueError("Too...")
+        raise ValueError(
+            "Too many filters - liblzma supports a maximum of %s" %
+            m.LZMA_FILTERS_MAX)
     filters = ffi.new('lzma_filter[]', m.LZMA_FILTERS_MAX+1)
     _owns[filters] = children = []
     for i in range(m.LZMA_FILTERS_MAX+1):
@@ -241,7 +243,7 @@
     elif filter.id in BCJ_FILTERS:
         add_opts('lzma_options_bcj', 'start_offset')
     else:
-        raise ValueError("Invalid...")
+        raise ValueError("Invalid filter ID: %s" % filter.id)
     return spec
 
 def _decode_filter_properties(filter_id, encoded_props):
@@ -425,25 +427,26 @@
 
     For one-shot decompression, use the decompress() function instead.
     """
-    def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None, header=None, check=None, unpadded_size=None):
+    def __init__(self, format=FORMAT_AUTO, memlimit=None, filters=None,
+                 header=None, check=None, unpadded_size=None):
         decoder_flags = m.LZMA_TELL_ANY_CHECK | m.LZMA_TELL_NO_CHECK
-        #decoder_flags = 0
         if memlimit is not None:
             if format == FORMAT_RAW:
-                raise ValueError("Cannot sp...")
-            #memlimit = long(memlimit)
+                raise ValueError("Cannot specify memory limit with FORMAT_RAW")
         else:
             memlimit = m.UINT64_MAX
 
         if format == FORMAT_RAW and filters is None:
-            raise ValueError("Must...")
+            raise ValueError("Must specify filters for FORMAT_RAW")
         elif format != FORMAT_RAW and filters is not None:
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify filters except with FORMAT_RAW")
 
         if format == FORMAT_BLOCK and (header is None or unpadded_size is None or check is None):
-            raise ValueError("Must...")
+            raise ValueError("Must specify header, unpadded_size and check "
+                             "with FORMAT_BLOCK")
         elif format != FORMAT_BLOCK and (header is not None or unpadded_size is not None or check is not None):
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify header, unpadded_size or check "
+                             "except with FORMAT_BLOCK")
 
         format = _parse_format(format)
         self.lock = threading.Lock()
@@ -481,7 +484,7 @@
             self.expected_size = block.compressed_size
             catch_lzma_error(m.lzma_block_decoder, self.lzs, block)
         else:
-            raise ValueError("invalid...")
+            raise ValueError("invalid container format: %s" % format)
 
     def pre_decompress_left_data(self, buf, buf_size):
         # in this case there is data left that needs to be processed before the first
@@ -556,7 +559,7 @@
             raise TypeError("max_length parameter object cannot be interpreted as an integer")
         with self.lock:
             if self.eof:
-                raise EOFError("Already...")
+                raise EOFError("Already at end of stream")
             lzs = self.lzs
             data = to_bytes(data)
             buf = ffi.new('uint8_t[]', data)
@@ -694,9 +697,9 @@
     """
     def __init__(self, format=FORMAT_XZ, check=-1, preset=None, filters=None):
         if format != FORMAT_XZ and check not in (-1, m.LZMA_CHECK_NONE):
-            raise ValueError("Integrity...")
+            raise ValueError("Integrity checks are only supported by FORMAT_XZ")
         if preset is not None and filters is not None:
-            raise ValueError("Cannot...")
+            raise ValueError("Cannot specify both preset and filter chain")
         if preset is None:
             preset = m.LZMA_PRESET_DEFAULT
         format = _parse_format(format)
@@ -718,19 +721,19 @@
             if filters is None:
                 options = ffi.new('lzma_options_lzma*')
                 if m.lzma_lzma_preset(options, preset):
-                    raise LZMAError("Invalid...")
+                    raise LZMAError("Invalid compression preset: %s" % preset)
                 catch_lzma_error(m.lzma_alone_encoder, self.lzs,
                     options)
             else:
                 raise NotImplementedError
         elif format == FORMAT_RAW:
             if filters is None:
-                raise ValueError("Must...")
+                raise ValueError("Must specify filters for FORMAT_RAW")
             filters = parse_filter_chain_spec(filters)
             catch_lzma_error(m.lzma_raw_encoder, self.lzs,
                 filters)
         else:
-            raise ValueError("Invalid...")
+            raise ValueError("invalid container format: %s" % format)
 
     def compress(self, data):
         """
@@ -744,7 +747,7 @@
         """
         with self.lock:
             if self.flushed:
-                raise ValueError("Compressor...")
+                raise ValueError("Compressor has been flushed")
             return self._compress(data)
 
     def _compress(self, data, action=m.LZMA_RUN):
@@ -785,7 +788,7 @@
     def flush(self):
         with self.lock:
             if self.flushed:
-                raise ValueError("Repeated...")
+                raise ValueError("Repeated call to flush()")
             self.flushed = 1
             result = self._compress(b'', action=m.LZMA_FINISH)
             __pypy__.add_memory_pressure(-COMPRESSION_STREAM_SIZE)

From pypy.commits at gmail.com  Tue Jun 27 17:47:04 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 27 Jun 2017 14:47:04 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Fix translation on Windows. space.text_w
 returns an RPython bytestring.
Message-ID: <5952d258.94da1c0a.f36c0.0d9d@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91656:f5d11f59787a
Date: 2017-06-27 23:46 +0200
http://bitbucket.org/pypy/pypy/changeset/f5d11f59787a/

Log:	Fix translation on Windows. space.text_w returns an RPython
	bytestring.

diff --git a/pypy/module/_cffi_backend/errorbox.py b/pypy/module/_cffi_backend/errorbox.py
--- a/pypy/module/_cffi_backend/errorbox.py
+++ b/pypy/module/_cffi_backend/errorbox.py
@@ -86,7 +86,7 @@
                 return
 
             w_text = self.space.call_function(w_done)
-            p = rffi.unicode2wcharp(self.space.text_w(w_text),
+            p = rffi.unicode2wcharp(self.space.unicode_w(w_text),
                                     track_allocation=False)
             if self.text_p:
                 rffi.free_wcharp(self.text_p, track_allocation=False)

From pypy.commits at gmail.com  Wed Jun 28 02:00:54 2017
From: pypy.commits at gmail.com (amauryfa)
Date: Tue, 27 Jun 2017 23:00:54 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Oops, fix win32 build
Message-ID: <59534616.0e8b1c0a.988ee.5433@mx.google.com>

Author: Amaury Forgeot d'Arc <amauryfa at gmail.com>
Branch: py3.5
Changeset: r91657:d0f683da0c05
Date: 2017-06-28 07:59 +0200
http://bitbucket.org/pypy/pypy/changeset/d0f683da0c05/

Log:	Oops, fix win32 build

diff --git a/rpython/rlib/rposix.py b/rpython/rlib/rposix.py
--- a/rpython/rlib/rposix.py
+++ b/rpython/rlib/rposix.py
@@ -1301,7 +1301,7 @@
         newpath = _as_bytes0(newpath)
         handle_posix_error('link', c_link(oldpath, newpath))
     else:
-        traits = _preferred_traits(path1)
+        traits = _preferred_traits(oldpath)
         win32traits = make_win32_traits(traits)
         oldpath = traits.as_str0(oldpath)
         newpath = traits.as_str0(newpath)

From pypy.commits at gmail.com  Wed Jun 28 12:23:07 2017
From: pypy.commits at gmail.com (rlamy)
Date: Wed, 28 Jun 2017 09:23:07 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: Implement _PyNamespace_New()
Message-ID: <5953d7eb.d4931c0a.53696.e149@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91658:97ca3ac43c30
Date: 2017-06-28 17:22 +0100
http://bitbucket.org/pypy/pypy/changeset/97ca3ac43c30/

Log:	Implement _PyNamespace_New()

diff --git a/pypy/module/cpyext/__init__.py b/pypy/module/cpyext/__init__.py
--- a/pypy/module/cpyext/__init__.py
+++ b/pypy/module/cpyext/__init__.py
@@ -75,6 +75,7 @@
 import pypy.module.cpyext.methodobject
 import pypy.module.cpyext.dictproxyobject
 import pypy.module.cpyext.genobject
+import pypy.module.cpyext.namespaceobject
 
 # now that all rffi_platform.Struct types are registered, configure them
 api.configure_types()
diff --git a/pypy/module/cpyext/namespaceobject.py b/pypy/module/cpyext/namespaceobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/namespaceobject.py
@@ -0,0 +1,8 @@
+from pypy.module.cpyext.api import cts
+
+ at cts.decl("PyObject * _PyNamespace_New(PyObject *kwds)")
+def _PyNamespace_new(space, w_kwds):
+    return space.appexec([w_kwds], """(kwds):
+        from _structseq import SimpleNamespace
+        return SimpleNamespace(**kwds)
+        """)
diff --git a/pypy/module/cpyext/test/test_namespaceobject.py b/pypy/module/cpyext/test/test_namespaceobject.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/cpyext/test/test_namespaceobject.py
@@ -0,0 +1,11 @@
+from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+
+class AppTestNamespace(AppTestCpythonExtensionBase):
+    def test_simple(self):
+        from types import SimpleNamespace
+        module = self.import_extension('ns', [
+            ("new", "METH_O",
+             """
+                return _PyNamespace_New(args);
+             """)])
+        assert module.new({'a': 1, 'b': 2}) == SimpleNamespace(a=1, b=2)

From pypy.commits at gmail.com  Thu Jun 29 05:06:07 2017
From: pypy.commits at gmail.com (arigo)
Date: Thu, 29 Jun 2017 02:06:07 -0700 (PDT)
Subject: [pypy-commit] pypy.org extradoc: update the values
Message-ID: <5954c2ff.4d8edf0a.dd95d.2345@mx.google.com>

Author: Armin Rigo <arigo at tunes.org>
Branch: extradoc
Changeset: r894:7f0b7f8ad4a6
Date: 2017-06-29 11:05 +0200
http://bitbucket.org/pypy/pypy.org/changeset/7f0b7f8ad4a6/

Log:	update the values

diff --git a/don1.html b/don1.html
--- a/don1.html
+++ b/don1.html
@@ -9,13 +9,13 @@
 <script>
   $(function() {
     $("#progressbar").progressbar({
-      value: 63.8
+      value: 63.9
    });
   });
 </script>
 
    <!-- Income:PyPy:Donations:Py3k / 0.95 -->
-   $67020 of $105000 (63.8%)
+   $67077 of $105000 (63.9%)
    <div id="progressbar">
    </div>
 
@@ -23,7 +23,7 @@
   <li>
   <span id="donate_text">This donation goes towards supporting Python 3 in PyPy.
   Current status:
-    we have $2804 left
+    we have $2856 left
   in the account.</br> <a href="py3donate.html">Read proposal</a></span>
   </li>
   <li>

From pypy.commits at gmail.com  Thu Jun 29 14:26:22 2017
From: pypy.commits at gmail.com (rlamy)
Date: Thu, 29 Jun 2017 11:26:22 -0700 (PDT)
Subject: [pypy-commit] pypy default: Don't use magical api object in
 pypy.module.cpyext.test.test_number
Message-ID: <5955464e.0298df0a.70693.c192@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91659:5a8f2cdedcd0
Date: 2017-06-29 19:25 +0100
http://bitbucket.org/pypy/pypy/changeset/5a8f2cdedcd0/

Log:	Don't use magical api object in pypy.module.cpyext.test.test_number

diff --git a/pypy/module/cpyext/number.py b/pypy/module/cpyext/number.py
--- a/pypy/module/cpyext/number.py
+++ b/pypy/module/cpyext/number.py
@@ -95,53 +95,58 @@
 def func_rename(newname):
     return lambda func: func_with_new_name(func, newname)
 
-def make_numbermethod(name, spacemeth):
+def make_numbermethod(cname, spacemeth):
     @cpython_api([PyObject, PyObject], PyObject)
-    @func_rename('PyNumber_%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1, w_o2):
         meth = getattr(space, spacemeth)
         return meth(w_o1, w_o2)
+    return PyNumber_Method
 
 def make_unary_numbermethod(name, spacemeth):
     @cpython_api([PyObject], PyObject)
-    @func_rename('PyNumber_%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1):
         meth = getattr(space, spacemeth)
         return meth(w_o1)
+    return PyNumber_Method
 
-def make_inplace_numbermethod(name, spacemeth):
+def make_inplace_numbermethod(cname, spacemeth):
     spacemeth = 'inplace_' + spacemeth.rstrip('_')
     @cpython_api([PyObject, PyObject], PyObject)
-    @func_rename('PyNumber_InPlace%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1, w_o2):
         meth = getattr(space, spacemeth)
         return meth(w_o1, w_o2)
+    return PyNumber_Method
 
 for name, spacemeth in [
-    ('Add', 'add'),
-    ('Subtract', 'sub'),
-    ('Multiply', 'mul'),
-    ('Divide', 'div'),
-    ('FloorDivide', 'floordiv'),
-    ('TrueDivide', 'truediv'),
-    ('Remainder', 'mod'),
-    ('Lshift', 'lshift'),
-    ('Rshift', 'rshift'),
-    ('And', 'and_'),
-    ('Xor', 'xor'),
-    ('Or', 'or_'),
-    ('Divmod', 'divmod'),
-    ]:
-    make_numbermethod(name, spacemeth)
+        ('Add', 'add'),
+        ('Subtract', 'sub'),
+        ('Multiply', 'mul'),
+        ('Divide', 'div'),
+        ('FloorDivide', 'floordiv'),
+        ('TrueDivide', 'truediv'),
+        ('Remainder', 'mod'),
+        ('Lshift', 'lshift'),
+        ('Rshift', 'rshift'),
+        ('And', 'and_'),
+        ('Xor', 'xor'),
+        ('Or', 'or_'),
+        ('Divmod', 'divmod')]:
+    cname = 'PyNumber_%s' % (name,)
+    globals()[cname] = make_numbermethod(cname, spacemeth)
     if name != 'Divmod':
-        make_inplace_numbermethod(name, spacemeth)
+        cname = 'PyNumber_InPlace%s' % (name,)
+        globals()[cname] = make_inplace_numbermethod(cname, spacemeth)
 
 for name, spacemeth in [
-    ('Negative', 'neg'),
-    ('Positive', 'pos'),
-    ('Absolute', 'abs'),
-    ('Invert', 'invert')]:
-    make_unary_numbermethod(name, spacemeth)
+        ('Negative', 'neg'),
+        ('Positive', 'pos'),
+        ('Absolute', 'abs'),
+        ('Invert', 'invert')]:
+    cname = 'PyNumber_%s' % (name,)
+    globals()[cname] = make_unary_numbermethod(cname, spacemeth)
 
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PyNumber_Power(space, w_o1, w_o2, w_o3):
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -1,53 +1,64 @@
 import py
+import pytest
 from rpython.rtyper.lltypesystem import lltype
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
-from pypy.module.cpyext.pyobject import PyObjectP, from_ref, make_ref, Py_DecRef
+from pypy.module.cpyext.pyobject import (
+    PyObjectP, from_ref, make_ref, Py_DecRef)
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.number import (
+    PyIndex_Check, PyNumber_Check, PyNumber_Long, PyNumber_Int,
+    PyNumber_Index, PyNumber_Coerce, PyNumber_CoerceEx, PyNumber_Add,
+    PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute,
+    PyNumber_Power, PyNumber_InPlacePower)
+from pypy.module.cpyext.floatobject import PyFloat_Check
+from pypy.module.cpyext.intobject import PyInt_CheckExact
+from pypy.module.cpyext.longobject import PyLong_CheckExact
+from pypy.module.cpyext.object import PyObject_Size
 
 class TestIterator(BaseApiTest):
-    def test_check(self, space, api):
-        assert api.PyIndex_Check(space.wrap(12))
-        assert api.PyIndex_Check(space.wraplong(-12L))
-        assert not api.PyIndex_Check(space.wrap(12.1))
-        assert not api.PyIndex_Check(space.wrap('12'))
+    def test_check(self, space):
+        assert PyIndex_Check(space, space.wrap(12))
+        assert PyIndex_Check(space, space.wraplong(-12L))
+        assert not PyIndex_Check(space, space.wrap(12.1))
+        assert not PyIndex_Check(space, space.wrap('12'))
 
-        assert api.PyNumber_Check(space.wrap(12))
-        assert api.PyNumber_Check(space.wraplong(-12L))
-        assert api.PyNumber_Check(space.wrap(12.1))
-        assert not api.PyNumber_Check(space.wrap('12'))
-        assert api.PyNumber_Check(space.wrap(1+3j))
+        assert PyNumber_Check(space, space.wrap(12))
+        assert PyNumber_Check(space, space.wraplong(-12L))
+        assert PyNumber_Check(space, space.wrap(12.1))
+        assert not PyNumber_Check(space, space.wrap('12'))
+        assert PyNumber_Check(space, space.wrap(1 + 3j))
 
-    def test_number_long(self, space, api):
-        w_l = api.PyNumber_Long(space.wrap(123))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Long(space.wrap("123"))
-        assert api.PyLong_CheckExact(w_l)
+    def test_number_long(self, space):
+        w_l = PyNumber_Long(space, space.wrap(123))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Long(space, space.wrap("123"))
+        assert PyLong_CheckExact(space, w_l)
 
-    def test_number_int(self, space, api):
-        w_l = api.PyNumber_Int(space.wraplong(123L))
-        assert api.PyInt_CheckExact(w_l)
-        w_l = api.PyNumber_Int(space.wrap(2 << 65))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Int(space.wrap(42.3))
-        assert api.PyInt_CheckExact(w_l)
-        w_l = api.PyNumber_Int(space.wrap("42"))
-        assert api.PyInt_CheckExact(w_l)
+    def test_number_int(self, space):
+        w_l = PyNumber_Int(space, space.wraplong(123L))
+        assert PyInt_CheckExact(space, w_l)
+        w_l = PyNumber_Int(space, space.wrap(2 << 65))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Int(space, space.wrap(42.3))
+        assert PyInt_CheckExact(space, w_l)
+        w_l = PyNumber_Int(space, space.wrap("42"))
+        assert PyInt_CheckExact(space, w_l)
 
-    def test_number_index(self, space, api):
-        w_l = api.PyNumber_Index(space.wraplong(123L))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Index(space.wrap(42.3))
-        assert w_l is None
-        api.PyErr_Clear()
+    def test_number_index(self, space):
+        w_l = PyNumber_Index(space, space.wraplong(123L))
+        assert PyLong_CheckExact(space, w_l)
+        with pytest.raises(OperationError):
+            PyNumber_Index(space, space.wrap(42.3))
 
-    def test_coerce(self, space, api):
+    def test_coerce(self, space):
         w_obj1 = space.wrap(123)
         w_obj2 = space.wrap(456.789)
         pp1 = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         pp1[0] = make_ref(space, w_obj1)
         pp2 = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
         pp2[0] = make_ref(space, w_obj2)
-        assert api.PyNumber_Coerce(pp1, pp2) == 0
+        assert PyNumber_Coerce(space, pp1, pp2) == 0
         assert space.str_w(space.repr(from_ref(space, pp1[0]))) == '123.0'
         assert space.str_w(space.repr(from_ref(space, pp2[0]))) == '456.789'
         Py_DecRef(space, pp1[0])
@@ -58,7 +69,7 @@
         Py_DecRef(space, w_obj2)
         lltype.free(pp2, flavor='raw')
 
-    def test_number_coerce_ex(self, space, api):
+    def test_number_coerce_ex(self, space):
         pl = make_ref(space, space.wrap(123))
         pf = make_ref(space, space.wrap(42.))
         ppl = lltype.malloc(PyObjectP.TO, 1, flavor='raw')
@@ -66,12 +77,12 @@
         ppl[0] = pl
         ppf[0] = pf
 
-        ret = api.PyNumber_CoerceEx(ppl, ppf)
+        ret = PyNumber_CoerceEx(space, ppl, ppf)
         assert ret == 0
 
         w_res = from_ref(space, ppl[0])
 
-        assert api.PyFloat_Check(w_res)
+        assert PyFloat_Check(space, w_res)
         assert space.unwrap(w_res) == 123.
         Py_DecRef(space, pl)
         Py_DecRef(space, pf)
@@ -80,31 +91,31 @@
         lltype.free(ppl, flavor='raw')
         lltype.free(ppf, flavor='raw')
 
-    def test_numbermethods(self, space, api):
+    def test_numbermethods(self, space):
         assert "ab" == space.unwrap(
-            api.PyNumber_Add(space.wrap("a"), space.wrap("b")))
+            PyNumber_Add(space, space.wrap("a"), space.wrap("b")))
         assert "aaa" == space.unwrap(
-            api.PyNumber_Multiply(space.wrap("a"), space.wrap(3)))
+            PyNumber_Multiply(space, space.wrap("a"), space.wrap(3)))
 
         w_l = space.newlist([1, 2, 3])
-        w_l2 = api.PyNumber_Multiply(w_l, space.wrap(3))
-        assert api.PyObject_Size(w_l2) == 9
-        assert api.PyObject_Size(w_l) == 3
+        w_l2 = PyNumber_Multiply(space, w_l, space.wrap(3))
+        assert PyObject_Size(space, w_l2) == 9
+        assert PyObject_Size(space, w_l) == 3
 
-        w_l3 = api.PyNumber_InPlaceMultiply(w_l, space.wrap(3))
-        assert api.PyObject_Size(w_l) == 9
+        w_l3 = PyNumber_InPlaceMultiply(space, w_l, space.wrap(3))
+        assert PyObject_Size(space, w_l) == 9
         assert w_l3 is w_l
 
         # unary function
-        assert 9 == space.unwrap(api.PyNumber_Absolute(space.wrap(-9)))
+        assert 9 == space.unwrap(PyNumber_Absolute(space, space.wrap(-9)))
 
         # power
         assert 9 == space.unwrap(
-            api.PyNumber_Power(space.wrap(3), space.wrap(2), space.w_None))
+            PyNumber_Power(space, space.wrap(3), space.wrap(2), space.w_None))
         assert 4 == space.unwrap(
-            api.PyNumber_Power(space.wrap(3), space.wrap(2), space.wrap(5)))
+            PyNumber_Power(space, space.wrap(3), space.wrap(2), space.wrap(5)))
         assert 9 == space.unwrap(
-            api.PyNumber_InPlacePower(space.wrap(3), space.wrap(2), space.w_None))
+            PyNumber_InPlacePower(space, space.wrap(3), space.wrap(2), space.w_None))
 
 class AppTestCNumber(AppTestCpythonExtensionBase):
     def test_app_coerce(self):
@@ -145,7 +156,7 @@
         assert tupl[1] == 1.
         assert isinstance(tupl[0], float)'''
 
-    def test_PyNumber_Check(self):        
+    def test_PyNumber_Check(self):
         mod = self.import_extension('foo', [
             ("test_PyNumber_Check", "METH_VARARGS",
              '''

From pypy.commits at gmail.com  Fri Jun 30 08:50:55 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 30 Jun 2017 05:50:55 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-tcp-style-trx-length: Implement TCP style
 linear transaction length increase after abort,
 exponential increase only initially or after a constant amount of
 transactions
Message-ID: <5956492f.c7871c0a.d5a48.adaf@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-tcp-style-trx-length
Changeset: r2078:91207e4ad1b8
Date: 2017-06-26 12:22 +0200
http://bitbucket.org/pypy/stmgc/changeset/91207e4ad1b8/

Log:	Implement TCP style linear transaction length increase after abort,
	exponential increase only initially or after a constant amount of
	transactions

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -21,26 +21,32 @@
 #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000L
 // #define LARGE_FILL_MARK_NURSERY_BYTES   0x1000000000000000L
 
-#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.00000001)
+// corresponds to ~7 bytes nursery fill
+#define STM_MIN_RELATIVE_TRANSACTION_LENGTH (0.0000000001)
 
 static double get_new_transaction_length(stm_thread_local_t *tl, bool aborts) {
     const int multiplier = 100;
     double previous = tl->relative_transaction_length;
     double new = previous;
     if (aborts) {
-        tl->transaction_length_backoff = 3;
         if (previous > STM_MIN_RELATIVE_TRANSACTION_LENGTH) {
             new = previous / multiplier;
         } else {
-            new = 0;
+            new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
         }
+        // the shorter the trx, the more backoff
+        tl->transaction_length_backoff = (int)(1 / new);
+        tl->linear_transaction_length_increment = new;
     } else if (tl->transaction_length_backoff == 0) {
-        if (previous - (STM_MIN_RELATIVE_TRANSACTION_LENGTH * 0.1) < 0) {
-            new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
-        } else if (previous < 1) {
+        // backoff counter is zero, exponential increase up to 1
+        if (previous < 1) {
             new = previous * multiplier;
         }
     } else { // not abort and backoff != 0
+        // in backoff, linear increase up to 1
+        if (previous < 1) {
+            new = previous + tl->linear_transaction_length_increment;
+        }
         tl->transaction_length_backoff -= 1;
     }
     return new;
diff --git a/c8/stm/setup.c b/c8/stm/setup.c
--- a/c8/stm/setup.c
+++ b/c8/stm/setup.c
@@ -246,9 +246,10 @@
     tl->last_associated_segment_num = num + 1;
     tl->thread_local_counter = ++thread_local_counters;
 
-    /* init single thread mode */
+    /* init adaptive transaction length mode */
     tl->relative_transaction_length = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
     tl->transaction_length_backoff = 0;
+    tl->linear_transaction_length_increment = 0;
 
     *_get_cpth(tl) = pthread_self();
     _init_shadow_stack(tl);
diff --git a/c8/stmgc.h b/c8/stmgc.h
--- a/c8/stmgc.h
+++ b/c8/stmgc.h
@@ -92,8 +92,10 @@
     /* == adaptive single thread mode == */
     /* factor that is multiplied with max transaction length before the start of the next transaction on this thread */
     double relative_transaction_length;
-    /* when zero, transaction length may increase or decrease, otherwise transaction length may only decrease. is (re-)set to some value upon abort and counted down until zero upon successful validation. */
+    /* when zero, transaction length may increase exponentially, otherwise transaction length may only increase linearly. is (re-)set to some value upon abort and counted down until zero upon successful validation. */
     int transaction_length_backoff;
+    /* during the backoff, transaction length may increase linearly by this increment on every successful validation */
+    double linear_transaction_length_increment;
 } stm_thread_local_t;
 
 

From pypy.commits at gmail.com  Fri Jun 30 08:50:57 2017
From: pypy.commits at gmail.com (tobweber)
Date: Fri, 30 Jun 2017 05:50:57 -0700 (PDT)
Subject: [pypy-commit] stmgc c8-tcp-style-trx-length: Improve backoff
 duration computation
Message-ID: <59564931.e386df0a.7a4ff.7174@mx.google.com>

Author: Tobias Weber <tobias_weber89 at gmx.de>
Branch: c8-tcp-style-trx-length
Changeset: r2079:50a4a53200e3
Date: 2017-06-27 15:01 +0200
http://bitbucket.org/pypy/stmgc/changeset/50a4a53200e3/

Log:	Improve backoff duration computation

diff --git a/c8/stm/nursery.c b/c8/stm/nursery.c
--- a/c8/stm/nursery.c
+++ b/c8/stm/nursery.c
@@ -34,8 +34,8 @@
         } else {
             new = STM_MIN_RELATIVE_TRANSACTION_LENGTH;
         }
-        // the shorter the trx, the more backoff
-        tl->transaction_length_backoff = (int)(1 / new);
+        // the shorter the trx, the more backoff: 1000 at min trx length, proportional decrease to 1 at max trx length (think a/x + b = backoff)
+        tl->transaction_length_backoff = (int)(0.0000001 / new - 0.9999999);
         tl->linear_transaction_length_increment = new;
     } else if (tl->transaction_length_backoff == 0) {
         // backoff counter is zero, exponential increase up to 1

From pypy.commits at gmail.com  Fri Jun 30 15:43:55 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Jun 2017 12:43:55 -0700 (PDT)
Subject: [pypy-commit] pypy default: Kill test-only unwrapper_catch() and
 fix tests
Message-ID: <5956a9fb.e9addf0a.5c55b.3907@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91660:32f6ab17f6c1
Date: 2017-06-30 20:43 +0100
http://bitbucket.org/pypy/pypy/changeset/32f6ab17f6c1/

Log:	Kill test-only unwrapper_catch() and fix tests

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -448,38 +448,10 @@
             error=_compute_error(error, restype), gil=gil,
             result_borrowed=result_borrowed, result_is_ll=result_is_ll)
         FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
-
-        # ZZZ is this whole logic really needed???  It seems to be only
-        # for RPython code calling PyXxx() functions directly.  I would
-        # think that usually directly calling the function is clean
-        # enough now
-        def unwrapper_catch(space, *args):
-            try:
-                res = unwrapper(space, *args)
-            except OperationError as e:
-                if not hasattr(unwrapper.api_func, "error_value"):
-                    raise
-                state = space.fromcache(State)
-                state.set_exception(e)
-                if is_PyObject(restype):
-                    return None
-                else:
-                    return unwrapper.api_func.error_value
-            got_integer = isinstance(res, (int, long, float))
-            if isinstance(restype, lltype.Typedef):
-                real_restype = restype.OF
-            else:
-                real_restype = restype
-            expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                            rffi.cast(restype, 0) == 0)
-            assert got_integer == expect_integer, (
-                'got %r not integer' % (res,))
-            return res
-        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
-
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
+        INTERPLEVEL_API[func.__name__] = unwrapper  # used in tests
         return unwrapper
     return decorate
 
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -1,5 +1,8 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.iterator import PyIter_Next
 
 
 class TestIterator(BaseApiTest):
@@ -14,13 +17,12 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 1
         assert space.unwrap(api.PyIter_Next(w_iter)) == 2
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
-        assert api.PyIter_Next(w_iter) is None
-        assert not api.PyErr_Occurred()
+        assert PyIter_Next(space, w_iter) is None
 
-    def test_iternext_error(self,space, api):
-        assert api.PyIter_Next(space.w_None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+    def test_iternext_error(self, space):
+        with pytest.raises(OperationError) as excinfo:
+            PyIter_Next(space, space.w_None)
+        assert excinfo.value.w_type is space.w_TypeError
 
 
 class AppTestIterator(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -1,5 +1,8 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.listobject import PyList_Size
 
 class TestListObject(BaseApiTest):
     def test_list(self, space, api):
@@ -19,19 +22,19 @@
 
         assert not api.PyList_Check(space.newtuple([]))
         assert not api.PyList_CheckExact(space.newtuple([]))
-    
+
     def test_get_size(self, space, api):
         l = api.PyList_New(0)
         assert api.PyList_GET_SIZE(l) == 0
         api.PyList_Append(l, space.wrap(3))
         assert api.PyList_GET_SIZE(l) == 1
-    
-    def test_size(self, space, api):
+
+    def test_size(self, space):
         l = space.newlist([space.w_None, space.w_None])
-        assert api.PyList_Size(l) == 2
-        assert api.PyList_Size(space.w_None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        assert PyList_Size(space, l) == 2
+        with pytest.raises(OperationError) as excinfo:
+            PyList_Size(space, space.w_None)
+        assert excinfo.value.w_type is space.w_TypeError
 
     def test_insert(self, space, api):
         w_l = space.newlist([space.w_None, space.w_None])
@@ -49,7 +52,7 @@
         l = space.newlist([space.wrap(1), space.wrap(0), space.wrap(7000)])
         assert api.PyList_Sort(l) == 0
         assert space.eq_w(l, space.newlist([space.wrap(0), space.wrap(1), space.wrap(7000)]))
-    
+
     def test_reverse(self, space, api):
         l = space.newlist([space.wrap(3), space.wrap(2), space.wrap(1)])
         assert api.PyList_Reverse(l) == 0
@@ -128,9 +131,9 @@
         l = L([1])
         module.setlistitem(l, 0)
         assert len(l) == 1
-        
+
         raises(SystemError, module.setlistitem, (1, 2, 3), 0)
-    
+
         l = []
         module.appendlist(l, 14)
         assert len(l) == 1
@@ -250,7 +253,7 @@
                 old_count2 = Py_REFCNT(i2);
 
                 ret = PyList_Append(o, i1);
-                if (ret != 0) 
+                if (ret != 0)
                     return NULL;
                 /* check the result of Append(), and also force the list
                    to use the CPyListStrategy now */
@@ -281,7 +284,7 @@
                 PyList_GetItem(o, 0);
                 CHECKCOUNT(0, 0, "PyList_Get_Item");
 
-                Py_DECREF(o); 
+                Py_DECREF(o);
                 #ifndef PYPY_VERSION
                 {
                     // PyPy deletes only at teardown
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,11 +1,14 @@
-import sys, py
+import sys
+import pytest
+from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import maxint
-from pypy.objspace.std.intobject import W_IntObject
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from pypy.module.cpyext.longobject import (
+    PyLong_FromLong, PyLong_AsLong, PyLong_AsUnsignedLong, PyLong_AsLongLong,
+    PyLong_AsUnsignedLongLong, PyLong_AsUnsignedLongLongMask)
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
@@ -17,24 +20,25 @@
         assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == sys.maxint
 
-    def test_aslong(self, space, api):
-        w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+    def test_aslong(self, space):
+        w_value = PyLong_FromLong(space, (sys.maxint - 1) / 2)
         assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
         assert isinstance(w_value, W_LongObject)
-        value = api.PyLong_AsLong(w_value)
+        value = PyLong_AsLong(space, w_value)
         assert value == (sys.maxint - 1)
 
         w_value = space.mul(w_value, space.wrap(2))
-
-        value = api.PyLong_AsLong(w_value)
-        assert value == -1 and api.PyErr_Occurred() is space.w_OverflowError
-        api.PyErr_Clear()
-        value = api.PyLong_AsUnsignedLong(w_value)
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsLong(space, w_value)
+        assert excinfo.value.w_type is space.w_OverflowError
+        value = PyLong_AsUnsignedLong(space, w_value)
         assert value == (sys.maxint - 1) * 2
 
-        self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, space.wrap(-1))
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLong(space, space.newint(-1))
+        assert excinfo.value.w_type is space.w_OverflowError
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
@@ -69,20 +73,22 @@
         assert api.PyLong_Check(l)
         assert not api.PyLong_CheckExact(l)
 
-    def test_as_longlong(self, space, api):
-        assert api.PyLong_AsLongLong(space.wrap(1<<62)) == 1<<62
-        assert api.PyLong_AsLongLong(space.wrap(1<<63)) == -1
-        api.PyErr_Clear()
+    def test_as_longlong(self, space):
+        assert PyLong_AsLongLong(space, space.wrap(1 << 62)) == 1 << 62
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsLongLong(space, space.wrap(1 << 63))
+        assert excinfo.value.w_type is space.w_OverflowError
 
-        assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<63)) == 1<<63
-        assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<64)) == (1<<64) - 1
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyLong_AsUnsignedLongLong(space, space.wrap(1 << 63)) == 1 << 63
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLongLong(space, space.wrap(1 << 64))
+        assert excinfo.value.w_type is space.w_OverflowError
 
-        assert api.PyLong_AsUnsignedLongLongMask(
-            space.wrap(1<<64)) == 0
+        assert PyLong_AsUnsignedLongLongMask(space, space.wrap(1 << 64)) == 0
 
-        self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLongLong, space.wrap(-1))
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLongLong(space, space.newint(-1))
+        assert excinfo.value.w_type is space.w_OverflowError
 
     def test_as_long_and_overflow(self, space, api):
         overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, flavor='raw')
@@ -131,10 +137,6 @@
         assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         assert api._PyLong_Sign(space.wraplong(0L)) == 0
         assert api._PyLong_Sign(space.wraplong(2L)) == 1
         assert api._PyLong_Sign(space.wraplong(-2L)) == -1
@@ -282,7 +284,7 @@
                     return PyLong_FromLong(3);
                  if (str + strlen(str) != end)
                     return PyLong_FromLong(4);
-                 return PyLong_FromLong(0); 
+                 return PyLong_FromLong(0);
              """)])
         assert module.from_str() == 0
 
@@ -338,4 +340,4 @@
         assert module.has_pow() == 0
         assert module.has_hex() == '0x2aL'
         assert module.has_oct() == '052L'
-                
+
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -1,4 +1,6 @@
-from pypy.module.cpyext.modsupport import PyModule_New
+import pytest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.modsupport import PyModule_New, PyModule_GetName
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from rpython.rtyper.lltypesystem import rffi
 
@@ -10,10 +12,12 @@
         assert space.eq_w(space.getattr(w_mod, space.newtext('__name__')),
                           space.newtext('testname'))
 
-    def test_module_getname(self, space, api):
+    def test_module_getname(self, space):
         w_sys = space.wrap(space.sys)
-        p = api.PyModule_GetName(w_sys)
+        p = PyModule_GetName(space, w_sys)
         assert rffi.charp2str(p) == 'sys'
-        p2 = api.PyModule_GetName(w_sys)
+        p2 = PyModule_GetName(space, w_sys)
         assert p2 == p
-        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
+        with pytest.raises(OperationError) as excinfo:
+            PyModule_GetName(space, space.w_True)
+        assert excinfo.value.w_type is space.w_SystemError
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -1,11 +1,14 @@
-import py
+import pytest
 import os
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.descriptor import get_dtype_cache
 import pypy.module.micronumpy.constants as NPY
+from pypy.module.cpyext.ndarrayobject import (
+    _PyArray_FromAny, _PyArray_FromObject)
 
 def scalar(space):
     dtype = get_dtype_cache(space).w_float64dtype
@@ -87,19 +90,19 @@
         ptr = rffi.cast(rffi.DOUBLEP, api._PyArray_DATA(a))
         assert ptr[0] == 10.
 
-    def test_FromAny(self, space, api):
+    def test_FromAny(self, space):
         a = array(space, [10, 5, 3])
-        assert api._PyArray_FromAny(a, None, 0, 0, 0, NULL) is a
-        assert api._PyArray_FromAny(a, None, 1, 4, 0, NULL) is a
-        self.raises(space, api, ValueError, api._PyArray_FromAny,
-                    a, None, 4, 5, 0, NULL)
+        assert _PyArray_FromAny(space, a, None, 0, 0, 0, NULL) is a
+        assert _PyArray_FromAny(space, a, None, 1, 4, 0, NULL) is a
+        with pytest.raises(OperationError) as excinfo:
+            _PyArray_FromAny(space, a, None, 4, 5, 0, NULL)
 
-    def test_FromObject(self, space, api):
+    def test_FromObject(self, space):
         a = array(space, [10, 5, 3])
-        assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
-                    a, 11, 4, 5)
-        assert exc.errorstr(space).find('desired') >= 0
+        assert _PyArray_FromObject(space, a, a.get_dtype().num, 0, 0) is a
+        with pytest.raises(OperationError) as excinfo:
+            _PyArray_FromObject(space, a, 11, 4, 5)
+        assert excinfo.value.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
@@ -216,7 +219,7 @@
         assert res.get_scalar_value().imag == 4.
 
     def _test_Ufunc_FromFuncAndDataAndSignature(self, space, api):
-        py.test.skip('preliminary non-translated test')
+        pytest.skip('preliminary non-translated test')
         '''
         PyUFuncGenericFunction funcs[] = {&double_times2, &int_times2};
         char types[] = { NPY_DOUBLE,NPY_DOUBLE, NPY_INT, NPY_INT };
@@ -363,7 +366,7 @@
     def test_ufunc(self):
         if self.runappdirect:
             from numpy import arange
-            py.test.xfail('segfaults on cpython: PyUFunc_API == NULL?')
+            pytest.xfail('segfaults on cpython: PyUFunc_API == NULL?')
         else:
             from _numpypy.multiarray import arange
         mod = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -1,10 +1,17 @@
-import py, pytest
+import pytest
 
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.cpyext.api import (
     Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT)
+from pypy.module.cpyext.object import (
+    PyObject_IsTrue, PyObject_Not, PyObject_GetAttrString,
+    PyObject_DelAttrString, PyObject_GetAttr, PyObject_DelAttr,
+    PyObject_GetItem, PyObject_RichCompareBool,
+    PyObject_IsInstance, PyObject_IsSubclass, PyObject_AsFileDescriptor,
+    PyObject_Hash, PyObject_Cmp, PyObject_Unicode
+)
 
 class TestObject(BaseApiTest):
     def test_IsTrue(self, space, api):
@@ -25,9 +32,10 @@
                     raise ValueError
             return C()""")
 
-        assert api.PyObject_IsTrue(w_obj) == -1
-        assert api.PyObject_Not(w_obj) == -1
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyObject_IsTrue(space, w_obj)
+        with raises_w(space, ValueError):
+            PyObject_Not(space, w_obj)
 
     def test_HasAttr(self, space, api):
         hasattr_ = lambda w_obj, name: api.PyObject_HasAttr(w_obj,
@@ -59,22 +67,21 @@
         rffi.free_charp(buf)
         assert space.unwrap(space.getattr(w_obj, space.wrap('test'))) == 20
 
-    def test_getattr(self, space, api):
+    def test_getattr(self, space):
         charp1 = rffi.str2charp("__len__")
         charp2 = rffi.str2charp("not_real")
-        assert api.PyObject_GetAttrString(space.wrap(""), charp1)
-        assert not api.PyObject_GetAttrString(space.wrap(""), charp2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
-        assert api.PyObject_DelAttrString(space.wrap(""), charp1) == -1
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        assert PyObject_GetAttrString(space, space.wrap(""), charp1)
+
+        with raises_w(space, AttributeError):
+            PyObject_GetAttrString(space, space.wrap(""), charp2)
+        with raises_w(space, AttributeError):
+            PyObject_DelAttrString(space, space.wrap(""), charp1)
         rffi.free_charp(charp1)
         rffi.free_charp(charp2)
 
-        assert api.PyObject_GetAttr(space.wrap(""), space.wrap("__len__"))
-        assert api.PyObject_DelAttr(space.wrap(""), space.wrap("__len__")) == -1
-        api.PyErr_Clear()
+        assert PyObject_GetAttr(space, space.wrap(""), space.wrap("__len__"))
+        with raises_w(space, AttributeError):
+            PyObject_DelAttr(space, space.wrap(""), space.wrap("__len__"))
 
     def test_getitem(self, space, api):
         w_t = space.wrap((1, 2, 3, 4, 5))
@@ -88,9 +95,8 @@
         assert space.getitem(w_d, space.wrap("key")) is space.w_None
 
         assert api.PyObject_DelItem(w_d, space.wrap("key")) == 0
-        assert api.PyObject_GetItem(w_d, space.wrap("key")) is None
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        with raises_w(space, KeyError):
+            PyObject_GetItem(space, w_d, space.wrap("key"))
 
     def test_size(self, space, api):
         assert api.PyObject_Size(space.newlist([space.w_None])) == 1
@@ -119,9 +125,9 @@
             w_o2 = space.wrap(o2)
 
             for opid, expected in [
-                    (Py_LT, o1 <  o2), (Py_LE, o1 <= o2),
+                    (Py_LT, o1 < o2), (Py_LE, o1 <= o2),
                     (Py_NE, o1 != o2), (Py_EQ, o1 == o2),
-                    (Py_GT, o1 >  o2), (Py_GE, o1 >= o2)]:
+                    (Py_GT, o1 > o2), (Py_GE, o1 >= o2)]:
                 assert compare(w_o1, w_o2, opid) == expected
 
         test_compare(1, 2)
@@ -129,9 +135,8 @@
         test_compare('2', '1')
 
         w_i = space.wrap(1)
-        assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1
-        assert api.PyErr_Occurred() is space.w_SystemError
-        api.PyErr_Clear()
+        with raises_w(space, SystemError):
+            PyObject_RichCompareBool(space, w_i, w_i, 123456)
 
     def test_IsInstance(self, space, api):
         assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1
@@ -140,8 +145,8 @@
         assert api.PyObject_IsInstance(
             space.wrap(1), space.newtuple([space.w_int, space.w_float])) == 1
         assert api.PyObject_IsInstance(space.w_type, space.w_type) == 1
-        assert api.PyObject_IsInstance(space.wrap(1), space.w_None) == -1
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_IsInstance(space, space.wrap(1), space.w_None)
 
     def test_IsSubclass(self, space, api):
         assert api.PyObject_IsSubclass(space.w_type, space.w_type) == 1
@@ -149,14 +154,13 @@
         assert api.PyObject_IsSubclass(space.w_object, space.w_type) == 0
         assert api.PyObject_IsSubclass(
             space.w_type, space.newtuple([space.w_int, space.w_type])) == 1
-        assert api.PyObject_IsSubclass(space.wrap(1), space.w_type) == -1
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_IsSubclass(space, space.wrap(1), space.w_type)
 
     def test_fileno(self, space, api):
         assert api.PyObject_AsFileDescriptor(space.wrap(1)) == 1
-        assert api.PyObject_AsFileDescriptor(space.wrap(-20)) == -1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyObject_AsFileDescriptor(space, space.wrap(-20))
 
         w_File = space.appexec([], """():
             class File:
@@ -169,9 +173,8 @@
     def test_hash(self, space, api):
         assert api.PyObject_Hash(space.wrap(72)) == 72
         assert api.PyObject_Hash(space.wrap(-1)) == -2
-        assert (api.PyObject_Hash(space.wrap([])) == -1 and
-            api.PyErr_Occurred() is space.w_TypeError)
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_Hash(space, space.wrap([]))
 
     def test_hash_double(self, space, api):
         assert api._Py_HashDouble(72.0) == 72
@@ -191,16 +194,15 @@
             assert ptr[0] == -1
             assert api.PyObject_Cmp(w("a"), w("a"), ptr) == 0
             assert ptr[0] == 0
-            assert api.PyObject_Cmp(w(u"\xe9"), w("\xe9"), ptr) < 0
-            assert api.PyErr_Occurred()
-            api.PyErr_Clear()
+            with raises_w(space, UnicodeDecodeError):
+                PyObject_Cmp(space, w(u"\xe9"), w("\xe9"), ptr)
 
     def test_unicode(self, space, api):
         assert space.unwrap(api.PyObject_Unicode(None)) == u"<NULL>"
         assert space.unwrap(api.PyObject_Unicode(space.wrap([]))) == u"[]"
         assert space.unwrap(api.PyObject_Unicode(space.wrap("e"))) == u"e"
-        assert api.PyObject_Unicode(space.wrap("\xe9")) is None
-        api.PyErr_Clear()
+        with raises_w(space, UnicodeDecodeError):
+            PyObject_Unicode(space, space.wrap("\xe9"))
 
     def test_dir(self, space, api):
         w_dir = api.PyObject_Dir(space.sys)
@@ -218,7 +220,7 @@
         from pypy.interpreter import gateway
 
         AppTestCpythonExtensionBase.setup_class.im_func(cls)
-        tmpname = str(py.test.ensuretemp('out', dir=0))
+        tmpname = str(pytest.ensuretemp('out', dir=0))
         if cls.runappdirect:
             cls.tmpname = tmpname
         else:
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -5,9 +5,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, ll2ctypes
-
-from pypy.interpreter.gateway import interp2app
+from rpython.rtyper.lltypesystem import rffi
 
 class TestExceptions(BaseApiTest):
     def test_GivenExceptionMatches(self, space, api):
@@ -70,13 +68,6 @@
         assert space.eq_w(state.operror.w_type, space.w_MemoryError)
         api.PyErr_Clear()
 
-    def test_BadArgument(self, space, api):
-        ret = api.PyErr_BadArgument()
-        state = space.fromcache(State)
-        assert space.eq_w(state.operror.w_type, space.w_TypeError)
-        assert ret == 0
-        api.PyErr_Clear()
-
     def test_Warning(self, space, api, capfd):
         message = rffi.str2charp("this is a warning")
         api.PyErr_WarnEx(None, message, 1)
@@ -369,7 +360,7 @@
         "XXX seems to pass, but doesn't: 'py.test -s' shows errors in PyObject_Free")
     def test_GetSetExcInfo(self):
         import sys
-        if self.runappdirect and (sys.version_info.major < 3 or 
+        if self.runappdirect and (sys.version_info.major < 3 or
                                   sys.version_info.minor < 3):
             skip('PyErr_{GS}etExcInfo introduced in python 3.3')
         module = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py
--- a/pypy/module/cpyext/test/test_pystrtod.py
+++ b/pypy/module/cpyext/test/test_pystrtod.py
@@ -1,67 +1,62 @@
 import math
 
 from pypy.module.cpyext import pystrtod
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
+from pypy.module.cpyext.pystrtod import PyOS_string_to_double
 
 
 class TestPyOS_string_to_double(BaseApiTest):
 
-    def test_simple_float(self, api):
+    def test_simple_float(self, space):
         s = rffi.str2charp('0.4')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert r == 0.4
         rffi.free_charp(s)
 
-    def test_empty_string(self, api):
+    def test_empty_string(self, space):
         s = rffi.str2charp('')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, None)
         rffi.free_charp(s)
 
-    def test_bad_string(self, api):
+    def test_bad_string(self, space):
         s = rffi.str2charp(' 0.4')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, None)
         rffi.free_charp(s)
 
-    def test_overflow_pos(self, api):
+    def test_overflow_pos(self, space):
         s = rffi.str2charp('1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert math.isinf(r)
         assert r > 0
         rffi.free_charp(s)
 
-    def test_overflow_neg(self, api):
+    def test_overflow_neg(self, space):
         s = rffi.str2charp('-1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert math.isinf(r)
         assert r < 0
         rffi.free_charp(s)
 
-    def test_overflow_exc(self, space, api):
+    def test_overflow_exc(self, space):
         s = rffi.str2charp('1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, space.w_ValueError)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, space.w_ValueError)
         rffi.free_charp(s)
 
-    def test_endptr_number(self, api):
+    def test_endptr_number(self, space):
         s = rffi.str2charp('0.4')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
+        r = PyOS_string_to_double(space, s, endp, None)
         assert r == 0.4
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
@@ -69,10 +64,10 @@
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
-    def test_endptr_tail(self, api):
+    def test_endptr_tail(self, space):
         s = rffi.str2charp('0.4 foo')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
+        r = PyOS_string_to_double(space, s, endp, None)
         assert r == 0.4
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
@@ -80,16 +75,14 @@
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
-    def test_endptr_no_conversion(self, api):
+    def test_endptr_no_conversion(self, space):
         s = rffi.str2charp('foo')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
-        assert r == -1.0
-        raises(ValueError)
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, endp, None)
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
         assert endp_addr == s_addr
-        api.PyErr_Clear()
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
@@ -164,4 +157,4 @@
         r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype)
         assert '3.14' == rffi.charp2str(r)
         assert ptype == lltype.nullptr(rffi.INTP.TO)
-        rffi.free_charp(r)
\ No newline at end of file
+        rffi.free_charp(r)
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -1,9 +1,12 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.interpreter.error import OperationError
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext import sequence
-import py.test
+from pypy.module.cpyext.sequence import (
+    PySequence_Fast, PySequence_Contains, PySequence_Index,
+    PySequence_GetItem, PySequence_SetItem, PySequence_DelItem)
+
+import pytest
 
 class TestSequence(BaseApiTest):
     def test_check(self, space, api):
@@ -56,16 +59,12 @@
         w_t2 = api.PySequence_InPlaceRepeat(w_t1, 3)
         assert space.unwrap(w_t2) == [0, 1, 0, 1, 0, 1]
 
-    def test_exception(self, space, api):
+    def test_exception(self, space):
         message = rffi.str2charp("message")
-        assert not api.PySequence_Fast(space.wrap(3), message)
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
-
-        exc = raises(OperationError, sequence.PySequence_Fast,
-                     space, space.wrap(3), message)
-        assert exc.value.match(space, space.w_TypeError)
-        assert space.str_w(exc.value.get_w_value(space)) == "message"
+        with pytest.raises(OperationError) as excinfo:
+            PySequence_Fast(space, space.wrap(3), message)
+        assert excinfo.value.match(space, space.w_TypeError)
+        assert space.str_w(excinfo.value.get_w_value(space)) == "message"
         rffi.free_charp(message)
 
     def test_get_slice(self, space, api):
@@ -80,7 +79,7 @@
 
     def test_get_slice_fast(self, space, api):
         w_t = space.wrap([1, 2, 3, 4, 5])
-        api.PySequence_Fast(w_t, "foo") # converts
+        api.PySequence_Fast(w_t, "foo")  # converts
         assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
         assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
 
@@ -97,13 +96,12 @@
         exc = raises(OperationError, space.next, w_iter)
         assert exc.value.match(space, space.w_StopIteration)
 
-    def test_contains(self, space, api):
+    def test_contains(self, space):
         w_t = space.wrap((1, 'ha'))
-        assert api.PySequence_Contains(w_t, space.wrap(u'ha'))
-        assert not api.PySequence_Contains(w_t, space.wrap(2))
-        assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PySequence_Contains(space, w_t, space.wrap(u'ha'))
+        assert not PySequence_Contains(space, w_t, space.wrap(2))
+        with raises_w(space, TypeError):
+            PySequence_Contains(space, space.w_None, space.wrap(2))
 
     def test_setitem(self, space, api):
         w_value = space.wrap(42)
@@ -112,39 +110,33 @@
         result = api.PySequence_SetItem(l, 0, w_value)
         assert result != -1
         assert space.eq_w(space.getitem(l, space.wrap(0)), w_value)
-
-        self.raises(space, api, IndexError, api.PySequence_SetItem,
-                    l, 3, w_value)
+        with raises_w(space, IndexError):
+            PySequence_SetItem(space, l, 3, w_value)
 
         t = api.PyTuple_New(1)
         api.PyTuple_SetItem(t, 0, l)
-        self.raises(space, api, TypeError, api.PySequence_SetItem,
-                    t, 0, w_value)
-
-        self.raises(space, api, TypeError, api.PySequence_SetItem,
-                    space.newdict(), 0, w_value)
+        with raises_w(space, TypeError):
+            PySequence_SetItem(space, t, 0, w_value)
+        with raises_w(space, TypeError):
+            PySequence_SetItem(space, space.newdict(), 0, w_value)
 
     def test_delitem(self, space, api):
         w_l = space.wrap([1, 2, 3, 4])
-
         result = api.PySequence_DelItem(w_l, 2)
         assert result == 0
         assert space.eq_w(w_l, space.wrap([1, 2, 4]))
-
-        self.raises(space, api, IndexError, api.PySequence_DelItem,
-                    w_l, 3)
+        with raises_w(space, IndexError):
+            PySequence_DelItem(space, w_l, 3)
 
     def test_getitem(self, space, api):
         thelist = [8, 7, 6, 5, 4, 3, 2, 1]
         w_l = space.wrap(thelist)
-
         result = api.PySequence_GetItem(w_l, 4)
         assert space.is_true(space.eq(result, space.wrap(4)))
-
         result = api.PySequence_ITEM(w_l, 4)
         assert space.is_true(space.eq(result, space.wrap(4)))
-
-        self.raises(space, api, IndexError, api.PySequence_GetItem, w_l, 9000)
+        with raises_w(space, IndexError):
+            PySequence_GetItem(space, w_l, 9000)
 
     def test_index(self, space, api):
         thelist = [9, 8, 7, 6, 5, 4, 3, 2, 1]
@@ -155,10 +147,8 @@
         assert result == thelist.index(5)
 
         w_tofind = space.wrap(9001)
-        result = api.PySequence_Index(w_l, w_tofind)
-        assert result == -1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PySequence_Index(space, w_l, w_tofind)
 
         w_gen = space.appexec([], """():
            return (x ** 2 for x in range(40))""")
@@ -196,7 +186,7 @@
         assert space.int_w(space.len(w_l)) == 4
         assert space.int_w(space.getitem(w_l, space.wrap(1))) == 2
         assert space.int_w(space.getitem(w_l, space.wrap(0))) == 1
-        e = py.test.raises(OperationError, space.getitem, w_l, space.wrap(15))
+        e = pytest.raises(OperationError, space.getitem, w_l, space.wrap(15))
         assert "list index out of range" in e.value.errorstr(space)
         assert space.int_w(space.getitem(w_l, space.wrap(-1))) == 4
         space.setitem(w_l, space.wrap(1), space.wrap(13))
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -1,31 +1,28 @@
-import py
-
-from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.setobject import PySet_Add, PySet_Size
 
 
 class TestTupleObject(BaseApiTest):
     def test_setobj(self, space, api):
         assert not api.PySet_Check(space.w_None)
         assert not api.PyFrozenSet_Check(space.w_None)
-        assert api.PySet_Add(space.w_None, space.w_None) == -1
-        api.PyErr_Clear()
+        with raises_w(space, SystemError):
+            PySet_Add(space, space.w_None, space.w_None)
         w_set = space.call_function(space.w_set)
         assert not api.PyFrozenSet_CheckExact(w_set)
-        space.call_method(w_set, 'update', space.wrap([1,2,3,4]))
+        space.call_method(w_set, 'update', space.wrap([1, 2, 3, 4]))
         assert api.PySet_Size(w_set) == 4
         assert api.PySet_GET_SIZE(w_set) == 4
-        raises(TypeError, api.PySet_Size(space.newlist([])))
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PySet_Size(space, space.newlist([]))
 
     def test_set_add_discard(self, space, api):
         w_set = api.PySet_New(None)
         assert api.PySet_Size(w_set) == 0
-        w_set = api.PyFrozenSet_New(space.wrap([1,2,3,4]))
+        w_set = api.PyFrozenSet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Size(w_set) == 4
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Size(w_set) == 4
         api.PySet_Add(w_set, space.wrap(6))
         assert api.PySet_Size(w_set) == 5
@@ -33,14 +30,14 @@
         assert api.PySet_Size(w_set) == 4
 
     def test_set_contains(self, space, api):
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Contains(w_set, space.wrap(1))
         assert not api.PySet_Contains(w_set, space.wrap(0))
 
     def test_set_pop_clear(self, space, api):
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         w_obj = api.PySet_Pop(w_set)
-        assert space.int_w(w_obj) in (1,2,3,4)
+        assert space.int_w(w_obj) in (1, 2, 3, 4)
         assert space.len_w(w_set) == 3
         api.PySet_Clear(w_set)
         assert space.len_w(w_set) == 0
@@ -72,6 +69,5 @@
              PySet_GET_SIZE(dumb_pointer);
 
              return o;
-             """
-            )
+             """)
         ])
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -1,24 +1,26 @@
 import py
 
 from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref
-from pypy.module.cpyext.tupleobject import PyTupleObject
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import FatalError
+from pypy.module.cpyext.tupleobject import (
+    PyTupleObject, PyTuple_SetItem, PyTuple_Size)
 
 
 class TestTupleObject(BaseApiTest):
 
     def test_tupleobject(self, space, api):
         assert not api.PyTuple_Check(space.w_None)
-        assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1
+        with raises_w(space, SystemError):
+            PyTuple_SetItem(space, space.w_None, 0, space.w_None)
         atuple = space.newtuple([space.wrap(0), space.wrap(1),
                                  space.wrap('yay')])
         assert api.PyTuple_Size(atuple) == 3
         #assert api.PyTuple_GET_SIZE(atuple) == 3  --- now a C macro
-        raises(TypeError, api.PyTuple_Size(space.newlist([])))
-        api.PyErr_Clear()
+        with raises_w(space, SystemError):
+            PyTuple_Size(space, space.newlist([]))
 
     def test_tuple_realize_refuses_nulls(self, space, api):
         py_tuple = api.PyTuple_New(1)
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test/test_weakref.py
--- a/pypy/module/cpyext/test/test_weakref.py
+++ b/pypy/module/cpyext/test/test_weakref.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.weakrefobject import PyWeakref_NewRef
 
 class TestWeakReference(BaseApiTest):
     def test_weakref(self, space, api):
@@ -10,12 +11,11 @@
         assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj)
 
         w_obj = space.newtuple([])
-        assert api.PyWeakref_NewRef(w_obj, space.w_None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyWeakref_NewRef(space, w_obj, space.w_None)
 
     def test_proxy(self, space, api):
-        w_obj = space.w_Warning # some weakrefable object
+        w_obj = space.w_Warning  # some weakrefable object
         w_proxy = api.PyWeakref_NewProxy(w_obj, None)
         assert space.unwrap(space.str(w_proxy)) == "<type 'exceptions.Warning'>"
         assert space.unwrap(space.repr(w_proxy)).startswith('<weak')

From pypy.commits at gmail.com  Fri Jun 30 17:03:43 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Jun 2017 14:03:43 -0700 (PDT)
Subject: [pypy-commit] pypy default: Change signature of PyXXX_Check
 pseudo-macros from int(PyObject*) to int(void*) (fixes #2596)
Message-ID: <5956bcaf.a494df0a.97a27.6b83@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: 
Changeset: r91661:36eb150b3c67
Date: 2017-06-30 22:03 +0100
http://bitbucket.org/pypy/pypy/changeset/36eb150b3c67/

Log:	Change signature of PyXXX_Check pseudo-macros from int(PyObject*) to
	int(void*) (fixes #2596)

diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -727,22 +727,21 @@
             return space.gettypeobject(cls.typedef)
     check_name = "Py" + type_name + "_Check"
 
+    @cts.decl("int %s(void * obj)" % check_name, error=CANNOT_FAIL)
     def check(space, w_obj):
         "Implements the Py_Xxx_Check function"
         w_obj_type = space.type(w_obj)
         w_type = get_w_type(space)
         return (space.is_w(w_obj_type, w_type) or
                 space.issubtype_w(w_obj_type, w_type))
+
+    @cts.decl("int %sExact(void * obj)" % check_name, error=CANNOT_FAIL)
     def check_exact(space, w_obj):
         "Implements the Py_Xxx_CheckExact function"
         w_obj_type = space.type(w_obj)
         w_type = get_w_type(space)
         return space.is_w(w_obj_type, w_type)
 
-    check = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)(
-        func_with_new_name(check, check_name))
-    check_exact = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)(
-        func_with_new_name(check_exact, check_name + "Exact"))
     return check, check_exact
 
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -95,3 +95,15 @@
 
              Py_RETURN_NONE;"""),
             ])
+
+    def test_PyFloat_Check(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+             PyObject* pyobj = PyFloat_FromDouble(1.0);
+             PyFloatObject* pfo = (PyFloatObject*)pyobj;
+             int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) &&
+                PyFloat_Check(pfo) && PyFloat_CheckExact(pfo);
+             return PyLong_FromLong(res);"""),
+            ])
+        assert module.test() == 1
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -1,8 +1,7 @@
-import pytest
-from pypy.interpreter.error import OperationError
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.listobject import PyList_Size
+from pypy.module.cpyext.listobject import (
+    PyList_Check, PyList_CheckExact, PyList_Size)
 
 class TestListObject(BaseApiTest):
     def test_list(self, space, api):
@@ -13,15 +12,15 @@
         """)
 
         l = api.PyList_New(0)
-        assert api.PyList_Check(l)
-        assert api.PyList_CheckExact(l)
+        assert PyList_Check(space, l)
+        assert PyList_CheckExact(space, l)
 
         l = space.call_function(L)
-        assert api.PyList_Check(l)
-        assert not api.PyList_CheckExact(l)
+        assert PyList_Check(space, l)
+        assert not PyList_CheckExact(space, l)
 
-        assert not api.PyList_Check(space.newtuple([]))
-        assert not api.PyList_CheckExact(space.newtuple([]))
+        assert not PyList_Check(space, space.newtuple([]))
+        assert not PyList_CheckExact(space, space.newtuple([]))
 
     def test_get_size(self, space, api):
         l = api.PyList_New(0)
@@ -32,9 +31,8 @@
     def test_size(self, space):
         l = space.newlist([space.w_None, space.w_None])
         assert PyList_Size(space, l) == 2
-        with pytest.raises(OperationError) as excinfo:
+        with raises_w(space, TypeError):
             PyList_Size(space, space.w_None)
-        assert excinfo.value.w_type is space.w_TypeError
 
     def test_insert(self, space, api):
         w_l = space.newlist([space.w_None, space.w_None])
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -6,7 +6,7 @@
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.longobject import (
+from pypy.module.cpyext.longobject import (PyLong_Check, PyLong_CheckExact,
     PyLong_FromLong, PyLong_AsLong, PyLong_AsUnsignedLong, PyLong_AsLongLong,
     PyLong_AsUnsignedLongLong, PyLong_AsUnsignedLongLongMask)
 
@@ -57,12 +57,12 @@
 
     def test_type_check(self, space, api):
         w_l = space.wrap(sys.maxint + 1)
-        assert api.PyLong_Check(w_l)
-        assert api.PyLong_CheckExact(w_l)
+        assert PyLong_Check(space, w_l)
+        assert PyLong_CheckExact(space, w_l)
 
         w_i = space.wrap(sys.maxint)
-        assert not api.PyLong_Check(w_i)
-        assert not api.PyLong_CheckExact(w_i)
+        assert not PyLong_Check(space, w_i)
+        assert not PyLong_CheckExact(space, w_i)
 
         L = space.appexec([], """():
             class L(long):
@@ -70,8 +70,8 @@
             return L
         """)
         l = space.call_function(L)
-        assert api.PyLong_Check(l)
-        assert not api.PyLong_CheckExact(l)
+        assert PyLong_Check(space, l)
+        assert not PyLong_CheckExact(space, l)
 
     def test_as_longlong(self, space):
         assert PyLong_AsLongLong(space, space.wrap(1 << 62)) == 1 << 62
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,10 +1,11 @@
+import pytest
 from pypy.conftest import option
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.tool.udir import udir
-import pytest
+from pypy.module.cpyext.pyfile import PyFile_Check, PyFile_CheckExact
 
 class TestFile(BaseApiTest):
 
@@ -15,9 +16,9 @@
         rffi.free_charp(filename)
         rffi.free_charp(mode)
 
-        assert api.PyFile_Check(w_file)
-        assert api.PyFile_CheckExact(w_file)
-        assert not api.PyFile_Check(space.wrap("text"))
+        assert PyFile_Check(space, w_file)
+        assert PyFile_CheckExact(space, w_file)
+        assert not PyFile_Check(space, space.wrap("text"))
 
         space.call_method(w_file, "write", space.newbytes("text"))
         space.call_method(w_file, "close")
@@ -70,7 +71,7 @@
                 assert fp is not None
                 w_file2 = api.PyFile_FromFile(fp, filename, mode, None)
         assert w_file2 is not None
-        assert api.PyFile_Check(w_file2)
+        assert PyFile_Check(space, w_file2)
         assert space.str_w(api.PyFile_Name(w_file2)) == name
 
     @pytest.mark.xfail
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -1,19 +1,21 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.setobject import PySet_Add, PySet_Size
+from pypy.module.cpyext.setobject import (
+    PySet_Check, PyFrozenSet_Check, PyFrozenSet_CheckExact,
+    PySet_Add, PySet_Size, PySet_GET_SIZE)
 
 
 class TestTupleObject(BaseApiTest):
-    def test_setobj(self, space, api):
-        assert not api.PySet_Check(space.w_None)
-        assert not api.PyFrozenSet_Check(space.w_None)
+    def test_setobj(self, space):
+        assert not PySet_Check(space, space.w_None)
+        assert not PyFrozenSet_Check(space, space.w_None)
         with raises_w(space, SystemError):
             PySet_Add(space, space.w_None, space.w_None)
         w_set = space.call_function(space.w_set)
-        assert not api.PyFrozenSet_CheckExact(w_set)
+        assert not PyFrozenSet_CheckExact(space, w_set)
         space.call_method(w_set, 'update', space.wrap([1, 2, 3, 4]))
-        assert api.PySet_Size(w_set) == 4
-        assert api.PySet_GET_SIZE(w_set) == 4
+        assert PySet_Size(space, w_set) == 4
+        assert PySet_GET_SIZE(space, w_set) == 4
         with raises_w(space, TypeError):
             PySet_Size(space, space.newlist([]))
 
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py
--- a/pypy/module/cpyext/test/test_sliceobject.py
+++ b/pypy/module/cpyext/test/test_sliceobject.py
@@ -2,13 +2,14 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP
+from pypy.module.cpyext.sliceobject import PySlice_Check
 
 class TestSliceObject(BaseApiTest):
-    def test_slice(self, space, api):
+    def test_slice(self, space):
         w_i = space.wrap(10)
         w_slice = space.newslice(w_i, w_i, w_i)
-        assert api.PySlice_Check(w_slice)
-        assert not api.PySlice_Check(w_i)
+        assert PySlice_Check(space, w_slice)
+        assert not PySlice_Check(space, w_i)
 
     def test_GetIndicesEx(self, space, api):
         w = space.wrap
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -6,19 +6,18 @@
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import FatalError
 from pypy.module.cpyext.tupleobject import (
-    PyTupleObject, PyTuple_SetItem, PyTuple_Size)
+    PyTupleObject, PyTuple_Check, PyTuple_SetItem, PyTuple_Size)
 
 
 class TestTupleObject(BaseApiTest):
 
-    def test_tupleobject(self, space, api):
-        assert not api.PyTuple_Check(space.w_None)
+    def test_tupleobject(self, space):
+        assert not PyTuple_Check(space, space.w_None)
         with raises_w(space, SystemError):
             PyTuple_SetItem(space, space.w_None, 0, space.w_None)
         atuple = space.newtuple([space.wrap(0), space.wrap(1),
                                  space.wrap('yay')])
-        assert api.PyTuple_Size(atuple) == 3
-        #assert api.PyTuple_GET_SIZE(atuple) == 3  --- now a C macro
+        assert PyTuple_Size(space, atuple) == 3
         with raises_w(space, SystemError):
             PyTuple_Size(space, space.newlist([]))
 

From pypy.commits at gmail.com  Fri Jun 30 18:25:46 2017
From: pypy.commits at gmail.com (rlamy)
Date: Fri, 30 Jun 2017 15:25:46 -0700 (PDT)
Subject: [pypy-commit] pypy py3.5: hg merge default
Message-ID: <5956cfea.0395df0a.89121.bc74@mx.google.com>

Author: Ronan Lamy <ronan.lamy at gmail.com>
Branch: py3.5
Changeset: r91662:96e56ffe7aa7
Date: 2017-06-30 23:25 +0100
http://bitbucket.org/pypy/pypy/changeset/96e56ffe7aa7/

Log:	hg merge default

diff too long, truncating to 2000 out of 2002 lines

diff --git a/lib_pypy/stackless.py b/lib_pypy/stackless.py
--- a/lib_pypy/stackless.py
+++ b/lib_pypy/stackless.py
@@ -268,12 +268,22 @@
         assert abs(d) == 1
         source = getcurrent()
         source.tempval = arg
-        if d > 0:
-            cando = self.balance < 0
-            dir = d
-        else:
-            cando = self.balance > 0
-            dir = 0
+        while True:
+            if d > 0:
+                cando = self.balance < 0
+                dir = d
+            else:
+                cando = self.balance > 0
+                dir = 0
+
+            if cando and self.queue[0]._tasklet_killed:
+                # issue #2595: the tasklet was killed while waiting.
+                # drop that tasklet from consideration and try again.
+                self.balance += d
+                self.queue.popleft()
+            else:
+                # normal path
+                break
 
         if _channel_callback is not None:
             _channel_callback(self, source, dir, not cando)
@@ -348,6 +358,8 @@
     module.
     """
     tempval = None
+    _tasklet_killed = False
+
     def __new__(cls, func=None, label=''):
         res = coroutine.__new__(cls)
         res.label = label
@@ -395,6 +407,7 @@
         If the exception passes the toplevel frame of the tasklet,
         the tasklet will silently die.
         """
+        self._tasklet_killed = True
         if not self.is_zombie:
             # Killing the tasklet by throwing TaskletExit exception.
             coroutine.kill(self)
diff --git a/pypy/module/cppyy/backend/create_cppyy_package.py b/pypy/module/cppyy/backend/create_cppyy_package.py
new file mode 100755
--- /dev/null
+++ b/pypy/module/cppyy/backend/create_cppyy_package.py
@@ -0,0 +1,649 @@
+#!/usr/bin/env python
+from __future__ import print_function
+
+import os, sys
+import argparse, re, shutil, tarfile, urllib2
+
+
+DEBUG_TESTBUILD = False
+
+TARBALL_CACHE_DIR = 'releases'
+
+ROOT_KEEP = ['build', 'cmake', 'config', 'core', 'etc', 'interpreter',
+             'io', 'LICENSE', 'net', 'Makefile', 'CMakeLists.txt', 'math',
+             'main'] # main only needed in more recent root b/c of rootcling
+ROOT_CORE_KEEP = ['CMakeLists.txt', 'base', 'clib', 'clingutils', 'cont',
+                  'dictgen', 'foundation', 'lzma', 'macosx', 'meta',
+                  'metacling', 'metautils', 'rootcling_stage1', 'textinput',
+                  'thread', 'unix', 'utils', 'winnt', 'zip']
+ROOT_IO_KEEP = ['CMakeLists.txt', 'io', 'rootpcm']
+ROOT_NET_KEEP = ['CMakeLists.txt', 'net']
+ROOT_MATH_KEEP = ['CMakeLists.txt', 'mathcore']
+ROOT_ETC_KEEP = ['Makefile.arch', 'class.rules', 'cmake', 'dictpch',
+                 'gdb-backtrace.sh', 'gitinfo.txt', 'helgrind-root.supp',
+                 'hostcert.conf', 'system.plugins-ios',
+                 'valgrind-root-python.supp', 'valgrind-root.supp', 'vmc']
+
+ROOT_EXPLICIT_REMOVE = ['core/base/v7', 'math/mathcore/v7', 'io/io/v7']
+
+
+ERR_RELEASE_NOT_FOUND = 2
+
+
+#
+## CLI arguments
+#
+class ReleaseValidation(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        if not re.match(r'6\.\d\d\.\d\d', value):
+            raise argparse.ArgumentTypeError(
+                "release number should of the form '6.dd.dd'")
+        setattr(namespace, self.dest, value)
+        return value
+
+parser = argparse.ArgumentParser(
+    description='Build PyPi package for cppyy containing the minimum of ROOT')
+parser.add_argument('-r', '--release', type=str, nargs='?',
+                    action=ReleaseValidation, help='ROOT release to use')
+
+args = parser.parse_args()
+
+
+#
+## ROOT source pull and cleansing
+#
+def clean_directory(directory, keeplist, trim_cmake=True):
+    removed_entries = []
+    for entry in os.listdir(directory):
+        if entry[0] == '.' or entry in keeplist:
+            continue
+        removed_entries.append(entry)
+        entry = os.path.join(directory, entry)
+        print('now removing', entry)
+        if os.path.isdir(entry):
+            shutil.rmtree(entry)
+        else:
+            os.remove(entry)
+
+    if not trim_cmake:
+        return
+
+    # now take the removed entries out of the CMakeLists.txt
+    if removed_entries:
+        inp = os.path.join(directory, 'CMakeLists.txt')
+        print('trimming', inp)
+        outp = inp+'.new'
+        new_cml = open(outp, 'w')
+        for line in open(inp).readlines():
+            if ('add_subdirectory' in line) or\
+               ('COMMAND' in line and 'copy' in line) or\
+               ('ROOT_ADD_TEST_SUBDIRECTORY' in line) or\
+               ('install(DIRECTORY' in line):
+                for sub in removed_entries:
+                    if sub in line:
+                        line = '#'+line
+                        break
+            new_cml.write(line)
+        new_cml.close()
+        os.rename(outp, inp)
+    else:
+        print('reusing existing %s/CMakeLists.txt' % (directory,))
+ 
+
+class ReleaseValidation(argparse.Action):
+    def __call__(self, parser, namespace, value, option_string=None):
+        if not re.match(r'6\.\d\d\.\d\d', value):
+            raise argparse.ArgumentTypeError(
+                "release number should of the form '6.dd.dd'")
+        setattr(namespace, self.dest, value)
+        return value
+
+parser = argparse.ArgumentParser(
+    description='Build PyPi package for cppyy containing the minimum of ROOT')
+parser.add_argument('-r', '--release', type=str, nargs='?',
+                    action=ReleaseValidation, help='ROOT release to use')
+
+args = parser.parse_args()
+
+if not os.path.exists(TARBALL_CACHE_DIR):
+    os.mkdir(TARBALL_CACHE_DIR)
+
+if args.release:
+  # use provided release
+    fn = 'root_v%s.source.tar.gz' % args.release
+    addr = 'https://root.cern.ch/download/'+fn
+    if not os.path.exists(os.path.join(TARBALL_CACHE_DIR, fn)):
+        try:
+            print('retrieving', fn)
+            resp = urllib2.urlopen(addr, fn)
+            out = open(os.path.join(TARBALL_CACHE_DIR, fn), 'wb')
+            out.write(resp.read())
+            out.close()
+        except urllib2.HTTPError:
+            print('release %s not found' % args.release)
+            sys.exit(ERR_RELEASE_NOT_FOUND)
+    else:
+        print('reusing', fn, 'from local directory')
+else:
+    print('provide release ... getting latest release is not yet implemented ...')
+    sys.exit(1)
+  # get latest and set fn, args.release, etc.
+
+# construct version for package
+args.version = ''
+testnext = False
+for c in args.release:
+    if testnext:
+        testnext = False
+        if c == '0':
+            continue
+    if c == '.':
+        testnext = True
+    args.version += c
+args.version += '.0'
+
+fn = os.path.join(TARBALL_CACHE_DIR, fn)
+pkgdir = os.path.join('root-'+args.release)
+if not os.path.exists(pkgdir):
+    print('now extracting', args.release)
+    tf = tarfile.TarFile.gzopen(fn)
+    tf.extractall()
+    tf.close()
+else:
+    print('reusing existing directory', pkgdir)
+
+# remove everything except for the listed set of libraries
+os.chdir(pkgdir)
+
+clean_directory(os.path.curdir, ROOT_KEEP)
+clean_directory('core',         ROOT_CORE_KEEP)
+clean_directory('etc',          ROOT_ETC_KEEP, trim_cmake=False)
+clean_directory('io',           ROOT_IO_KEEP)
+clean_directory('math',         ROOT_MATH_KEEP)
+clean_directory('net',          ROOT_NET_KEEP)
+
+
+# trim main (only need rootcling)
+print('trimming main')
+for entry in os.listdir('main/src'):
+    if entry != 'rootcling.cxx':
+        os.remove('main/src/'+entry)
+inp = 'main/CMakeLists.txt'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if ('ROOT_EXECUTABLE' in line or\
+        'SET_TARGET_PROPERTIES' in line) and\
+       not 'rootcling' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+
+# remove afterimage and ftgl explicitly
+print('trimming externals')
+for cmf in ['AfterImage', 'FTGL']:
+    os.remove('cmake/modules/Find%s.cmake' % (cmf,))
+inp = 'cmake/modules/SearchInstalledSoftware.cmake'
+outp = inp+'.new'
+now_stripping = False
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if '#---Check for ftgl if needed' == line[0:28] or\
+       '#---Check for AfterImage' == line[0:24]:
+        now_stripping = True
+    elif '#---Check' == line[0:9]:
+        now_stripping = False
+    if now_stripping:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+inp = 'cmake/modules/RootBuildOptions.cmake'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if 'ROOT_BUILD_OPTION(builtin_ftgl' in line or\
+       'ROOT_BUILD_OPTION(builtin_afterimage' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+
+# remove testing and examples
+print('trimming testing')
+inp = 'CMakeLists.txt'
+outp = inp+'.new'
+now_stripping = False
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if '#---Configure Testing using CTest' == line[0:33] or\
+       '#---hsimple.root' == line[0:16]:
+        now_stripping = True
+    elif '#---Packaging' == line[0:13] or\
+         '#---version' == line[0:11]:
+        now_stripping = False
+    if now_stripping:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+print('trimming RootCPack')
+inp = 'cmake/modules/RootCPack.cmake'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp):
+    if 'README.txt' in line:
+        line = '#'+line
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+# some more explicit removes:
+for dir_to_remove in ROOT_EXPLICIT_REMOVE:
+    try:
+        shutil.rmtree(dir_to_remove)
+    except OSError:
+        pass
+
+# special fixes
+inp = 'core/base/src/TVirtualPad.cxx'
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp):
+    if '#include "X3DBuffer.h"' == line[0:22]:
+        line = """//#include "X3DBuffer.h"
+typedef struct _x3d_sizeof_ {
+   int  numPoints;
+   int  numSegs;
+   int  numPolys;
+} Size3D;
+"""
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+inp = 'math/mathcore/src/Fitter.cxx'
+if os.path.exists(inp):
+    outp = inp+'.new'
+    new_cml = open(outp, 'w')
+    for line in open(inp):
+        if '#include "TF1.h"' in line:
+            continue
+        new_cml.write(line)
+    new_cml.close()
+    os.rename(outp, inp)
+
+# done
+os.chdir(os.path.pardir)
+
+# debugging: run a test build
+if DEBUG_TESTBUILD:
+    print('running a debug test build')
+    tb = "test_builddir"
+    if os.path.exists(tb):
+        shutil.rmtree(tb)
+    os.mkdir(tb)
+    os.chdir(tb)
+    os.system('cmake ../%s -DCMAKE_INSTALL_PREFIX=../install -Dminimal=ON -Dasimage=OFF' % pkgdir)
+    os.system('make -j 32')
+
+
+#
+## package creation
+#
+countdown = 0
+pidir = 'Package-'+args.release
+print('creating package', pidir)
+if not os.path.exists(pidir):
+    os.mkdir(pidir)
+os.chdir(pidir); countdown += 1
+
+print('creating LICENSE.txt')
+with open('LICENSE.txt', 'w') as outp:
+    outp.write("""There are three main parts:
+
+ LLVM: distributed under University of Illinois/NCSA Open Source License
+   https://opensource.org/licenses/UoI-NCSA.php
+ ROOT: distributed under LGPL 2.1
+   https://root.cern.ch/license
+ Cppyy: distributed under LBNL BSD
+   https://fedoraproject.org/wiki/Licensing/LBNLBSD
+""")
+
+print('creating MANIFEST.in')
+with open('MANIFEST.in', 'w') as outp:
+    outp.write("""# Include the license file
+include LICENSE.txt
+
+# Include the data files
+recursive-include src *
+""")
+
+print('creating README.rst')
+with open('README.rst', 'w') as outp:
+    outp.write("""PyPy cling-support
+==================
+
+----
+
+Find the documentation here:
+  http://doc.pypy.org/en/latest/cppyy.html
+""")
+
+print('creating setup.cfg')
+with open('setup.cfg', 'w') as outp:
+    outp.write("""[bdist_wheel]
+universal=0
+""")
+
+print('creating setup.py')
+with open('setup.py', 'w') as outp:
+    outp.write("""import os, sys, subprocess
+from setuptools import setup, find_packages
+from distutils import log
+from distutils.command.build import build as _build
+from setuptools.command.install import install as _install
+from distutils.sysconfig import get_python_lib
+from distutils.errors import DistutilsSetupError
+from codecs import open
+
+here = os.path.abspath(os.path.dirname(__file__))
+with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f:
+    long_description = f.read()
+
+builddir = None
+def get_builddir():
+    global builddir
+    if builddir is None:
+        topdir = os.getcwd()
+        builddir = os.path.join(topdir, 'builddir')
+    return builddir
+
+srcdir = None
+def get_srcdir():
+    global srcdir
+    if srcdir is None:
+        topdir = os.getcwd()
+        srcdir = os.path.join(topdir, 'src', 'backend')
+    return srcdir
+
+class my_cmake_build(_build):
+    def __init__(self, dist, *args, **kwargs):
+        _build.__init__(self, dist, *args, **kwargs)
+        # TODO: can't seem to find a better way of getting hold of
+        # the install_lib parameter during the build phase ...
+        prefix = ''
+        try:
+            prefix = dist.get_command_obj('install').install_lib
+        except AttributeError:
+            pass
+        if not prefix:
+            prefix = get_python_lib(1, 0)
+        self.prefix = os.path.join(prefix, 'cppyy_backend')
+
+    def run(self):
+        # base run
+        _build.run(self)
+
+        # custom run
+        log.info('Now building libcppyy_backend.so and dependencies')
+        builddir = get_builddir()
+        srcdir = get_srcdir()
+        if not os.path.exists(builddir):
+            log.info('Creating build directory %s ...' % builddir)
+            os.makedirs(builddir)
+
+        os.chdir(builddir)
+        log.info('Running cmake for cppyy_backend')
+        if subprocess.call([
+                'cmake', srcdir, '-Dminimal=ON -Dasimage=OFF',
+                '-DCMAKE_INSTALL_PREFIX='+self.prefix]) != 0:
+            raise DistutilsSetupError('Failed to configure cppyy_backend')
+
+        nprocs = os.getenv("MAKE_NPROCS")
+        if nprocs:
+            try:
+                ival = int(nprocs)
+                nprocs = '-j'+nprocs
+            except ValueError:
+                log.warn("Integer expected for MAKE_NPROCS, but got %s (ignored)", nprocs)
+                nprocs = '-j1'
+        else:
+            nprocs = '-j1'
+        log.info('Now building cppyy_backend and dependencies ...')
+        if subprocess.call(['make', nprocs]) != 0:
+            raise DistutilsSetupError('Failed to build cppyy_backend')
+
+        log.info('build finished')
+
+class my_libs_install(_install):
+    def run(self):
+        # base install
+        _install.run(self)
+
+        # custom install
+        log.info('Now installing libcppyy_backend.so and dependencies')
+        builddir = get_builddir()
+        if not os.path.exists(builddir):
+            raise DistutilsSetupError('Failed to find build dir!')
+        os.chdir(builddir)
+
+        prefix = self.install_lib
+        log.info('Now installing in %s ...', prefix)
+        if subprocess.call(['make', 'install']) != 0:
+            raise DistutilsSetupError('Failed to install cppyy_backend')
+
+        log.info('install finished')
+
+    def get_outputs(self):
+        outputs = _install.get_outputs(self)
+        outputs.append(os.path.join(self.install_lib, 'cppyy_backend'))
+        return outputs
+
+setup(
+    name='PyPy-cppyy-backend',
+""")
+    outp.write("    version='%s', # corresponds to ROOT %s, extra number is for packager\n"\
+         % (args.version, args.release))
+    outp.write("""    description='Cling support for PyPy',
+    long_description=long_description,
+
+    url='http://pypy.org',
+
+    # Author details
+    author='PyPy Developers',
+    author_email='pypy-dev at python.org',
+
+    license='LLVM: UoI-NCSA; ROOT: LGPL 2.1; Cppyy: LBNL BSD',
+
+    classifiers=[
+        'Development Status :: 4 - Beta',
+
+        'Intended Audience :: Developers',
+
+        'Topic :: Software Development',
+        'Topic :: Software Development :: Interpreters',
+
+        #'License :: OSI Approved :: MIT License',
+
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: Implementation :: PyPy',
+        'Programming Language :: C',
+        'Programming Language :: C++',
+
+        'Natural Language :: English'
+    ],
+
+    keywords='interpreter development',
+
+    packages=find_packages('src', ['backend']),
+    include_package_data=True,
+
+    extras_require={
+    },
+
+    cmdclass = {
+        'build': my_cmake_build,
+        'install': my_libs_install,
+    },
+)
+""")
+
+
+print('creating src ... ROOT part')
+if not os.path.exists('src'):
+    os.mkdir('src')
+os.chdir('src'); countdown += 1
+if not os.path.exists('backend'):
+    src = os.path.join(os.path.pardir, os.path.pardir, pkgdir)
+    print('now copying', src)
+    shutil.copytree(src, 'backend')
+
+print('creating src ... cppyy part')
+os.chdir('backend'); countdown += 1
+if not os.path.exists('cppyy'):
+    os.mkdir('cppyy')
+    os.chdir('cppyy'); countdown += 1
+
+    with open('CMakeLists.txt', 'w') as outp:
+        outp.write("""############################################################################
+# CMakeLists.txt file for building cppyy package
+############################################################################
+
+ROOT_GLOB_SOURCES(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cxx)
+set_source_files_properties(${sources} COMPILE_FLAGS "-fomit-frame-pointer -fvisibility=hidden -DRPY_EXTERN=RPY_EXPORTED -DRPYTHON_LL2CTYPES")
+
+add_definitions(${CLING_CXXFLAGS})
+
+ROOT_LINKER_LIBRARY(cppyy_backend ${sources}
+                               LIBRARIES ${CMAKE_DL_LIBS}
+                               DEPENDENCIES Core Cling RIO Thread)
+
+add_dependencies(cppyy_backend CLING)
+""")
+
+    os.mkdir('src')
+    os.chdir('src'); countdown += 1
+    print('pulling cppyy/clingcwrapper.cxx from pypy')
+    base = 'https://bitbucket.org/pypy/pypy/raw/default/pypy/module/cppyy/'
+    for cppyy_file in ['src/callcontext.h', 'include/capi.h', 'src/clingcwrapper.cxx',
+                       'include/clingcwrapper.h', 'include/cpp_cppyy.h', 'include/cppyy.h']:
+        resp = urllib2.urlopen(base+cppyy_file)
+        with open(os.path.basename(cppyy_file), 'w') as outp:
+            outp.write(resp.read())
+
+    # fix include
+    inp = 'capi.h'
+    outp = inp+'.new'
+    new_cml = open(outp, 'w')
+    for line in open(inp).readlines():
+        if 'src/precommondefs.h' in line:
+            line = '#include "precommondefs.h"\n'
+        new_cml.write(line)
+    new_cml.close()
+    os.rename(outp, inp)
+
+    with open('precommondefs.h', 'w') as outp:
+        outp.write("""/***** Start of precommondefs.h *****/
+
+/* This is extracted from pyconfig.h from CPython.  It sets the macros
+   that affect the features we get from system include files.
+   It must not #include anything. */
+
+#ifndef __PYPY_PRECOMMONDEFS_H
+#define __PYPY_PRECOMMONDEFS_H
+
+
+/* Define on Darwin to activate all library features */
+#define _DARWIN_C_SOURCE 1
+/* This must be set to 64 on some systems to enable large file support. */
+#define _FILE_OFFSET_BITS 64
+/* Define on Linux to activate all library features */
+#define _GNU_SOURCE 1
+/* This must be defined on some systems to enable large file support. */
+#define _LARGEFILE_SOURCE 1
+/* Define on NetBSD to activate all library features */
+#define _NETBSD_SOURCE 1
+/* Define to activate features from IEEE Stds 1003.1-2001 */
+#ifndef _POSIX_C_SOURCE
+#  define _POSIX_C_SOURCE 200112L
+#endif
+/* Define on FreeBSD to activate all library features */
+#define __BSD_VISIBLE 1
+#define __XSI_VISIBLE 700
+/* Windows: winsock/winsock2 mess */
+#define WIN32_LEAN_AND_MEAN
+#ifdef _WIN64
+   typedef          __int64 Signed;
+   typedef unsigned __int64 Unsigned;
+#  define SIGNED_MIN LLONG_MIN
+#else
+   typedef          long Signed;
+   typedef unsigned long Unsigned;
+#  define SIGNED_MIN LONG_MIN
+#endif
+
+#if !defined(RPY_ASSERT) && !defined(RPY_LL_ASSERT) && !defined(NDEBUG)
+#  define NDEBUG
+#endif
+
+
+/* All functions and global variables declared anywhere should use
+   one of the following attributes:
+
+   RPY_EXPORTED:  the symbol is exported out of libpypy-c.so.
+
+   RPY_EXTERN:    the symbol is not exported out of libpypy-c.so, but
+                  otherwise works like 'extern' by being available to
+                  other C sources.
+
+   static:        as usual, this means the symbol is local to this C file.
+
+   Don't use _RPY_HIDDEN directly.  For tests involving building a custom
+   .so, translator/tool/cbuild.py overrides RPY_EXTERN so that it becomes
+   equal to RPY_EXPORTED.
+
+   Any function or global variable declared with no attribute at all is
+   a bug; please report or fix it.
+*/
+#ifdef __GNUC__
+#  define RPY_EXPORTED extern __attribute__((visibility("default")))
+#  define _RPY_HIDDEN  __attribute__((visibility("hidden")))
+#else
+#  define RPY_EXPORTED extern __declspec(dllexport)
+#  define _RPY_HIDDEN  /* nothing */
+#endif
+#ifndef RPY_EXTERN
+#  define RPY_EXTERN   extern _RPY_HIDDEN
+#endif
+
+
+#endif /* __PYPY_PRECOMMONDEFS_H */
+
+/***** End of precommondefs.h *****/
+""")
+
+# back up to pip package top
+for i in range(countdown-1):
+    os.chdir(os.path.pardir)
+
+# add cppyy module to cmake
+os.chdir('src/backend')
+inp = 'CMakeLists.txt'
+print('adding cppyy to cmake')
+outp = inp+'.new'
+new_cml = open(outp, 'w')
+for line in open(inp).readlines():
+    if 'add_subdirectory' in line and 'net' in line:
+        line += 'add_subdirectory (cppyy)\n'
+    new_cml.write(line)
+new_cml.close()
+os.rename(outp, inp)
+
+# done!
diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py
--- a/pypy/module/cpyext/api.py
+++ b/pypy/module/cpyext/api.py
@@ -451,38 +451,10 @@
             error=_compute_error(error, restype), gil=gil,
             result_borrowed=result_borrowed, result_is_ll=result_is_ll)
         FUNCTIONS_BY_HEADER[header][func.__name__] = api_function
-
-        # ZZZ is this whole logic really needed???  It seems to be only
-        # for RPython code calling PyXxx() functions directly.  I would
-        # think that usually directly calling the function is clean
-        # enough now
-        def unwrapper_catch(space, *args):
-            try:
-                res = unwrapper(space, *args)
-            except OperationError as e:
-                if not hasattr(unwrapper.api_func, "error_value"):
-                    raise
-                state = space.fromcache(State)
-                state.set_exception(e)
-                if is_PyObject(restype):
-                    return None
-                else:
-                    return unwrapper.api_func.error_value
-            got_integer = isinstance(res, (int, long, float))
-            if isinstance(restype, lltype.Typedef):
-                real_restype = restype.OF
-            else:
-                real_restype = restype
-            expect_integer = (isinstance(real_restype, lltype.Primitive) and
-                            rffi.cast(restype, 0) == 0)
-            assert got_integer == expect_integer, (
-                'got %r not integer' % (res,))
-            return res
-        INTERPLEVEL_API[func.__name__] = unwrapper_catch  # used in tests
-
         unwrapper = api_function.get_unwrapper()
         unwrapper.func = func
         unwrapper.api_func = api_function
+        INTERPLEVEL_API[func.__name__] = unwrapper  # used in tests
         return unwrapper
     return decorate
 
@@ -769,22 +741,21 @@
             return space.gettypeobject(cls.typedef)
     check_name = "Py" + type_name + "_Check"
 
+    @cts.decl("int %s(void * obj)" % check_name, error=CANNOT_FAIL)
     def check(space, w_obj):
         "Implements the Py_Xxx_Check function"
         w_obj_type = space.type(w_obj)
         w_type = get_w_type(space)
         return (space.is_w(w_obj_type, w_type) or
                 space.issubtype_w(w_obj_type, w_type))
+
+    @cts.decl("int %sExact(void * obj)" % check_name, error=CANNOT_FAIL)
     def check_exact(space, w_obj):
         "Implements the Py_Xxx_CheckExact function"
         w_obj_type = space.type(w_obj)
         w_type = get_w_type(space)
         return space.is_w(w_obj_type, w_type)
 
-    check = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)(
-        func_with_new_name(check, check_name))
-    check_exact = cpython_api([PyObject], rffi.INT_real, error=CANNOT_FAIL)(
-        func_with_new_name(check_exact, check_name + "Exact"))
     return check, check_exact
 
 pypy_debug_catch_fatal_exception = rffi.llexternal('pypy_debug_catch_fatal_exception', [], lltype.Void)
diff --git a/pypy/module/cpyext/number.py b/pypy/module/cpyext/number.py
--- a/pypy/module/cpyext/number.py
+++ b/pypy/module/cpyext/number.py
@@ -55,54 +55,59 @@
 def func_rename(newname):
     return lambda func: func_with_new_name(func, newname)
 
-def make_numbermethod(name, spacemeth):
+def make_numbermethod(cname, spacemeth):
     @cpython_api([PyObject, PyObject], PyObject)
-    @func_rename('PyNumber_%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1, w_o2):
         meth = getattr(space, spacemeth)
         return meth(w_o1, w_o2)
+    return PyNumber_Method
 
 def make_unary_numbermethod(name, spacemeth):
     @cpython_api([PyObject], PyObject)
-    @func_rename('PyNumber_%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1):
         meth = getattr(space, spacemeth)
         return meth(w_o1)
+    return PyNumber_Method
 
-def make_inplace_numbermethod(name, spacemeth):
+def make_inplace_numbermethod(cname, spacemeth):
     spacemeth = 'inplace_' + spacemeth.rstrip('_')
     @cpython_api([PyObject, PyObject], PyObject)
-    @func_rename('PyNumber_InPlace%s' % (name,))
+    @func_rename(cname)
     def PyNumber_Method(space, w_o1, w_o2):
         meth = getattr(space, spacemeth)
         return meth(w_o1, w_o2)
+    return PyNumber_Method
 
 for name, spacemeth in [
-    ('Add', 'add'),
-    ('Subtract', 'sub'),
-    ('Multiply', 'mul'),
-    ('Divide', 'div'),
-    ('FloorDivide', 'floordiv'),
-    ('TrueDivide', 'truediv'),
-    ('Remainder', 'mod'),
-    ('Lshift', 'lshift'),
-    ('Rshift', 'rshift'),
-    ('And', 'and_'),
-    ('Xor', 'xor'),
-    ('Or', 'or_'),
-    ('Divmod', 'divmod'),
-    ('MatrixMultiply', 'matmul')
-    ]:
-    make_numbermethod(name, spacemeth)
+        ('Add', 'add'),
+        ('Subtract', 'sub'),
+        ('Multiply', 'mul'),
+        ('Divide', 'div'),
+        ('FloorDivide', 'floordiv'),
+        ('TrueDivide', 'truediv'),
+        ('Remainder', 'mod'),
+        ('Lshift', 'lshift'),
+        ('Rshift', 'rshift'),
+        ('And', 'and_'),
+        ('Xor', 'xor'),
+        ('Or', 'or_'),
+        ('Divmod', 'divmod'),
+        ('MatrixMultiply', 'matmul')]:
+    cname = 'PyNumber_%s' % (name,)
+    globals()[cname] = make_numbermethod(cname, spacemeth)
     if name != 'Divmod':
-        make_inplace_numbermethod(name, spacemeth)
+        cname = 'PyNumber_InPlace%s' % (name,)
+        globals()[cname] = make_inplace_numbermethod(cname, spacemeth)
 
 for name, spacemeth in [
-    ('Negative', 'neg'),
-    ('Positive', 'pos'),
-    ('Absolute', 'abs'),
-    ('Invert', 'invert')]:
-    make_unary_numbermethod(name, spacemeth)
+        ('Negative', 'neg'),
+        ('Positive', 'pos'),
+        ('Absolute', 'abs'),
+        ('Invert', 'invert')]:
+    cname = 'PyNumber_%s' % (name,)
+    globals()[cname] = make_unary_numbermethod(cname, spacemeth)
 
 @cpython_api([PyObject, PyObject, PyObject], PyObject)
 def PyNumber_Power(space, w_o1, w_o2, w_o3):
diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py
--- a/pypy/module/cpyext/test/test_floatobject.py
+++ b/pypy/module/cpyext/test/test_floatobject.py
@@ -95,3 +95,15 @@
 
              Py_RETURN_NONE;"""),
             ])
+
+    def test_PyFloat_Check(self):
+        module = self.import_extension('foo', [
+            ("test", "METH_NOARGS",
+             """
+             PyObject* pyobj = PyFloat_FromDouble(1.0);
+             PyFloatObject* pfo = (PyFloatObject*)pyobj;
+             int res = PyFloat_Check(pyobj) && PyFloat_CheckExact(pyobj) &&
+                PyFloat_Check(pfo) && PyFloat_CheckExact(pfo);
+             return PyLong_FromLong(res);"""),
+            ])
+        assert module.test() == 1
diff --git a/pypy/module/cpyext/test/test_iterator.py b/pypy/module/cpyext/test/test_iterator.py
--- a/pypy/module/cpyext/test/test_iterator.py
+++ b/pypy/module/cpyext/test/test_iterator.py
@@ -1,5 +1,8 @@
+import pytest
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.iterator import PyIter_Next
 
 
 class TestIterator(BaseApiTest):
@@ -14,13 +17,12 @@
         assert space.unwrap(api.PyIter_Next(w_iter)) == 1
         assert space.unwrap(api.PyIter_Next(w_iter)) == 2
         assert space.unwrap(api.PyIter_Next(w_iter)) == 3
-        assert api.PyIter_Next(w_iter) is None
-        assert not api.PyErr_Occurred()
+        assert PyIter_Next(space, w_iter) is None
 
-    def test_iternext_error(self,space, api):
-        assert api.PyIter_Next(space.w_None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+    def test_iternext_error(self, space):
+        with pytest.raises(OperationError) as excinfo:
+            PyIter_Next(space, space.w_None)
+        assert excinfo.value.w_type is space.w_TypeError
 
 
 class AppTestIterator(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_listobject.py b/pypy/module/cpyext/test/test_listobject.py
--- a/pypy/module/cpyext/test/test_listobject.py
+++ b/pypy/module/cpyext/test/test_listobject.py
@@ -1,6 +1,8 @@
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.pyobject import from_ref
+from pypy.module.cpyext.listobject import (
+    PyList_Check, PyList_CheckExact, PyList_Size)
 
 class TestListObject(BaseApiTest):
     def test_list(self, space, api):
@@ -11,15 +13,15 @@
         """)
 
         l = api.PyList_New(0)
-        assert api.PyList_Check(l)
-        assert api.PyList_CheckExact(l)
+        assert PyList_Check(space, l)
+        assert PyList_CheckExact(space, l)
 
         l = space.call_function(L)
-        assert api.PyList_Check(l)
-        assert not api.PyList_CheckExact(l)
+        assert PyList_Check(space, l)
+        assert not PyList_CheckExact(space, l)
 
-        assert not api.PyList_Check(space.newtuple([]))
-        assert not api.PyList_CheckExact(space.newtuple([]))
+        assert not PyList_Check(space, space.newtuple([]))
+        assert not PyList_CheckExact(space, space.newtuple([]))
 
     def test_get_size(self, space, api):
         l = api.PyList_New(0)
@@ -27,12 +29,11 @@
         api.PyList_Append(l, space.wrap(3))
         assert api.PyList_GET_SIZE(l) == 1
 
-    def test_size(self, space, api):
+    def test_size(self, space):
         l = space.newlist([space.w_None, space.w_None])
-        assert api.PyList_Size(l) == 2
-        assert api.PyList_Size(space.w_None) == -1
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        assert PyList_Size(space, l) == 2
+        with raises_w(space, TypeError):
+            PyList_Size(space, space.w_None)
 
     def test_insert(self, space, api):
         w_l = space.newlist([space.w_None, space.w_None])
diff --git a/pypy/module/cpyext/test/test_longobject.py b/pypy/module/cpyext/test/test_longobject.py
--- a/pypy/module/cpyext/test/test_longobject.py
+++ b/pypy/module/cpyext/test/test_longobject.py
@@ -1,10 +1,14 @@
-import sys, py
+import sys
+import pytest
+from pypy.interpreter.error import OperationError
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.rarithmetic import maxint
 from pypy.objspace.std.longobject import W_LongObject
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-
+from pypy.module.cpyext.longobject import (PyLong_Check, PyLong_CheckExact,
+    PyLong_FromLong, PyLong_AsLong, PyLong_AsUnsignedLong, PyLong_AsLongLong,
+    PyLong_AsUnsignedLongLong, PyLong_AsUnsignedLongLongMask)
 
 class TestLongObject(BaseApiTest):
     def test_FromLong(self, space, api):
@@ -16,24 +20,25 @@
         assert isinstance(w_value, W_LongObject)
         assert space.unwrap(w_value) == sys.maxint
 
-    def test_aslong(self, space, api):
-        w_value = api.PyLong_FromLong((sys.maxint - 1) / 2)
+    def test_aslong(self, space):
+        w_value = PyLong_FromLong(space, (sys.maxint - 1) / 2)
         assert isinstance(w_value, W_LongObject)
 
         w_value = space.mul(w_value, space.wrap(2))
         assert isinstance(w_value, W_LongObject)
-        value = api.PyLong_AsLong(w_value)
+        value = PyLong_AsLong(space, w_value)
         assert value == (sys.maxint - 1)
 
         w_value = space.mul(w_value, space.wrap(2))
-
-        value = api.PyLong_AsLong(w_value)
-        assert value == -1 and api.PyErr_Occurred() is space.w_OverflowError
-        api.PyErr_Clear()
-        value = api.PyLong_AsUnsignedLong(w_value)
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsLong(space, w_value)
+        assert excinfo.value.w_type is space.w_OverflowError
+        value = PyLong_AsUnsignedLong(space, w_value)
         assert value == (sys.maxint - 1) * 2
 
-        self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLong, space.wrap(-1))
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLong(space, space.newint(-1))
+        assert excinfo.value.w_type is space.w_OverflowError
 
     def test_as_ssize_t(self, space, api):
         w_value = space.newlong(2)
@@ -52,12 +57,12 @@
 
     def test_type_check(self, space, api):
         w_l = space.wrap(sys.maxint + 1)
-        assert api.PyLong_Check(w_l)
-        assert api.PyLong_CheckExact(w_l)
+        assert PyLong_Check(space, w_l)
+        assert PyLong_CheckExact(space, w_l)
 
         w_i = space.wrap(sys.maxint)
-        assert api.PyLong_Check(w_i)
-        assert api.PyLong_CheckExact(w_i)
+        assert PyLong_Check(space, w_i)
+        assert PyLong_CheckExact(space, w_i)
 
         L = space.appexec([], """():
             class L(int):
@@ -65,23 +70,25 @@
             return L
         """)
         l = space.call_function(L)
-        assert api.PyLong_Check(l)
-        assert not api.PyLong_CheckExact(l)
+        assert PyLong_Check(space, l)
+        assert not PyLong_CheckExact(space, l)
 
-    def test_as_longlong(self, space, api):
-        assert api.PyLong_AsLongLong(space.wrap(1<<62)) == 1<<62
-        assert api.PyLong_AsLongLong(space.wrap(1<<63)) == -1
-        api.PyErr_Clear()
+    def test_as_longlong(self, space):
+        assert PyLong_AsLongLong(space, space.wrap(1 << 62)) == 1 << 62
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsLongLong(space, space.wrap(1 << 63))
+        assert excinfo.value.w_type is space.w_OverflowError
 
-        assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<63)) == 1<<63
-        assert api.PyLong_AsUnsignedLongLong(space.wrap(1<<64)) == (1<<64) - 1
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PyLong_AsUnsignedLongLong(space, space.wrap(1 << 63)) == 1 << 63
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLongLong(space, space.wrap(1 << 64))
+        assert excinfo.value.w_type is space.w_OverflowError
 
-        assert api.PyLong_AsUnsignedLongLongMask(
-            space.wrap(1<<64)) == 0
+        assert PyLong_AsUnsignedLongLongMask(space, space.wrap(1 << 64)) == 0
 
-        self.raises(space, api, OverflowError, api.PyLong_AsUnsignedLongLong, space.wrap(-1))
+        with pytest.raises(OperationError) as excinfo:
+            PyLong_AsUnsignedLongLong(space, space.newint(-1))
+        assert excinfo.value.w_type is space.w_OverflowError
 
     def test_as_long_and_overflow(self, space, api):
         overflow = lltype.malloc(rffi.CArrayPtr(rffi.INT_real).TO, 1, flavor='raw')
@@ -126,10 +133,6 @@
         assert api.PyLong_AsVoidPtr(w_l) == p
 
     def test_sign_and_bits(self, space, api):
-        if space.is_true(space.lt(space.sys.get('version_info'),
-                                  space.wrap((2, 7)))):
-            py.test.skip("unsupported before Python 2.7")
-
         assert api._PyLong_Sign(space.wraplong(0L)) == 0
         assert api._PyLong_Sign(space.wraplong(2L)) == 1
         assert api._PyLong_Sign(space.wraplong(-2L)) == -1
@@ -328,6 +331,6 @@
                     ret = PyLong_FromLong(-1);
                 Py_DECREF(obj);
                 return ret;
-             """),])
+             """)])
         assert module.has_sub() == 0
         assert module.has_pow() == 0
diff --git a/pypy/module/cpyext/test/test_module.py b/pypy/module/cpyext/test/test_module.py
--- a/pypy/module/cpyext/test/test_module.py
+++ b/pypy/module/cpyext/test/test_module.py
@@ -1,4 +1,6 @@
-from pypy.module.cpyext.modsupport import PyModule_New
+import pytest
+from pypy.interpreter.error import OperationError
+from pypy.module.cpyext.modsupport import PyModule_New, PyModule_GetName
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi
@@ -11,13 +13,15 @@
         assert space.eq_w(space.getattr(w_mod, space.newtext('__name__')),
                           space.newtext('testname'))
 
-    def test_module_getname(self, space, api):
+    def test_module_getname(self, space):
         w_sys = space.wrap(space.sys)
-        p = api.PyModule_GetName(w_sys)
+        p = PyModule_GetName(space, w_sys)
         assert rffi.charp2str(p) == 'sys'
-        p2 = api.PyModule_GetName(w_sys)
+        p2 = PyModule_GetName(space, w_sys)
         assert p2 == p
-        self.raises(space, api, SystemError, api.PyModule_GetName, space.w_True)
+        with pytest.raises(OperationError) as excinfo:
+            PyModule_GetName(space, space.w_True)
+        assert excinfo.value.w_type is space.w_SystemError
 
 
 class AppTestModuleObject(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_ndarrayobject.py b/pypy/module/cpyext/test/test_ndarrayobject.py
--- a/pypy/module/cpyext/test/test_ndarrayobject.py
+++ b/pypy/module/cpyext/test/test_ndarrayobject.py
@@ -1,13 +1,16 @@
-import py
+import pytest
 import os
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 from pypy.module.micronumpy.ndarray import W_NDimArray
 from pypy.module.micronumpy.descriptor import get_dtype_cache
 import pypy.module.micronumpy.constants as NPY
+from pypy.module.cpyext.ndarrayobject import (
+    _PyArray_FromAny, _PyArray_FromObject)
 
-py.test.skip("Micronumpy not yet supported on py3k.")
+pytest.skip("Micronumpy not yet supported on py3k.")
 
 def scalar(space):
     dtype = get_dtype_cache(space).w_float64dtype
@@ -89,19 +92,19 @@
         ptr = rffi.cast(rffi.DOUBLEP, api._PyArray_DATA(a))
         assert ptr[0] == 10.
 
-    def test_FromAny(self, space, api):
+    def test_FromAny(self, space):
         a = array(space, [10, 5, 3])
-        assert api._PyArray_FromAny(a, None, 0, 0, 0, NULL) is a
-        assert api._PyArray_FromAny(a, None, 1, 4, 0, NULL) is a
-        self.raises(space, api, ValueError, api._PyArray_FromAny,
-                    a, None, 4, 5, 0, NULL)
+        assert _PyArray_FromAny(space, a, None, 0, 0, 0, NULL) is a
+        assert _PyArray_FromAny(space, a, None, 1, 4, 0, NULL) is a
+        with pytest.raises(OperationError) as excinfo:
+            _PyArray_FromAny(space, a, None, 4, 5, 0, NULL)
 
-    def test_FromObject(self, space, api):
+    def test_FromObject(self, space):
         a = array(space, [10, 5, 3])
-        assert api._PyArray_FromObject(a, a.get_dtype().num, 0, 0) is a
-        exc = self.raises(space, api, ValueError, api._PyArray_FromObject,
-                    a, 11, 4, 5)
-        assert exc.errorstr(space).find('desired') >= 0
+        assert _PyArray_FromObject(space, a, a.get_dtype().num, 0, 0) is a
+        with pytest.raises(OperationError) as excinfo:
+            _PyArray_FromObject(space, a, 11, 4, 5)
+        assert excinfo.value.errorstr(space).find('desired') >= 0
 
     def test_list_from_fixedptr(self, space, api):
         A = lltype.GcArray(lltype.Float)
@@ -218,7 +221,7 @@
         assert res.get_scalar_value().imag == 4.
 
     def _test_Ufunc_FromFuncAndDataAndSignature(self, space, api):
-        py.test.skip('preliminary non-translated test')
+        pytest.skip('preliminary non-translated test')
         '''
         PyUFuncGenericFunction funcs[] = {&double_times2, &int_times2};
         char types[] = { NPY_DOUBLE,NPY_DOUBLE, NPY_INT, NPY_INT };
@@ -365,7 +368,7 @@
     def test_ufunc(self):
         if self.runappdirect:
             from numpy import arange
-            py.test.xfail('segfaults on cpython: PyUFunc_API == NULL?')
+            pytest.xfail('segfaults on cpython: PyUFunc_API == NULL?')
         else:
             from _numpypy.multiarray import arange
         mod = self.import_extension('foo', [
diff --git a/pypy/module/cpyext/test/test_number.py b/pypy/module/cpyext/test/test_number.py
--- a/pypy/module/cpyext/test/test_number.py
+++ b/pypy/module/cpyext/test/test_number.py
@@ -1,68 +1,77 @@
+import pytest
 from rpython.rtyper.lltypesystem import lltype
+from pypy.interpreter.error import OperationError
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
+from pypy.module.cpyext.number import (
+    PyIndex_Check, PyNumber_Check, PyNumber_Long,
+    PyNumber_Index, PyNumber_Add,
+    PyNumber_Multiply, PyNumber_InPlaceMultiply, PyNumber_Absolute,
+    PyNumber_Power, PyNumber_InPlacePower)
+from pypy.module.cpyext.floatobject import PyFloat_Check
+from pypy.module.cpyext.longobject import PyLong_CheckExact
+from pypy.module.cpyext.object import PyObject_Size
 
 class TestIterator(BaseApiTest):
-    def test_check(self, space, api):
-        assert api.PyIndex_Check(space.wrap(12))
-        assert api.PyIndex_Check(space.wraplong(-12L))
-        assert not api.PyIndex_Check(space.wrap(12.1))
-        assert not api.PyIndex_Check(space.wrap('12'))
+    def test_check(self, space):
+        assert PyIndex_Check(space, space.wrap(12))
+        assert PyIndex_Check(space, space.wraplong(-12L))
+        assert not PyIndex_Check(space, space.wrap(12.1))
+        assert not PyIndex_Check(space, space.wrap('12'))
 
-        assert api.PyNumber_Check(space.wrap(12))
-        assert api.PyNumber_Check(space.wraplong(-12L))
-        assert api.PyNumber_Check(space.wrap(12.1))
-        assert not api.PyNumber_Check(space.wrap('12'))
-        assert api.PyNumber_Check(space.wrap(1+3j))
+        assert PyNumber_Check(space, space.wrap(12))
+        assert PyNumber_Check(space, space.wraplong(-12L))
+        assert PyNumber_Check(space, space.wrap(12.1))
+        assert not PyNumber_Check(space, space.wrap('12'))
+        assert PyNumber_Check(space, space.wrap(1 + 3j))
 
-    def test_number_long(self, space, api):
-        w_l = api.PyNumber_Long(space.wrap(123))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Long(space.wrap("123"))
-        assert api.PyLong_CheckExact(w_l)
+    def test_number_long(self, space):
+        w_l = PyNumber_Long(space, space.wrap(123))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Long(space, space.wrap("123"))
+        assert PyLong_CheckExact(space, w_l)
 
-    def test_number_long2(self, space, api):
-        w_l = api.PyNumber_Long(space.wraplong(123L))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Long(space.wrap(2 << 65))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Long(space.wrap(42.3))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Long(space.wrap("42"))
-        assert api.PyLong_CheckExact(w_l)
+    def test_number_long2(self, space):
+        w_l = PyNumber_Long(space, space.wraplong(123L))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Long(space, space.wrap(2 << 65))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Long(space, space.wrap(42.3))
+        assert PyLong_CheckExact(space, w_l)
+        w_l = PyNumber_Long(space, space.wrap("42"))
+        assert PyLong_CheckExact(space, w_l)
 
-    def test_number_index(self, space, api):
-        w_l = api.PyNumber_Index(space.wraplong(123L))
-        assert api.PyLong_CheckExact(w_l)
-        w_l = api.PyNumber_Index(space.wrap(42.3))
-        assert w_l is None
-        api.PyErr_Clear()
+    def test_number_index(self, space):
+        w_l = PyNumber_Index(space, space.wraplong(123L))
+        assert PyLong_CheckExact(space, w_l)
+        with pytest.raises(OperationError):
+            PyNumber_Index(space, space.wrap(42.3))
 
     def test_numbermethods(self, space, api):
         assert "ab" == space.unwrap(
-            api.PyNumber_Add(space.wrap("a"), space.wrap("b")))
+            PyNumber_Add(space, space.wrap("a"), space.wrap("b")))
         assert "aaa" == space.unwrap(
-            api.PyNumber_Multiply(space.wrap("a"), space.wrap(3)))
+            PyNumber_Multiply(space, space.wrap("a"), space.wrap(3)))
 
         w_l = space.newlist([1, 2, 3])
-        w_l2 = api.PyNumber_Multiply(w_l, space.wrap(3))
-        assert api.PyObject_Size(w_l2) == 9
-        assert api.PyObject_Size(w_l) == 3
+        w_l2 = PyNumber_Multiply(space, w_l, space.wrap(3))
+        assert PyObject_Size(space, w_l2) == 9
+        assert PyObject_Size(space, w_l) == 3
 
-        w_l3 = api.PyNumber_InPlaceMultiply(w_l, space.wrap(3))
-        assert api.PyObject_Size(w_l) == 9
+        w_l3 = PyNumber_InPlaceMultiply(space, w_l, space.wrap(3))
+        assert PyObject_Size(space, w_l) == 9
         assert w_l3 is w_l
 
         # unary function
-        assert 9 == space.unwrap(api.PyNumber_Absolute(space.wrap(-9)))
+        assert 9 == space.unwrap(PyNumber_Absolute(space, space.wrap(-9)))
 
         # power
         assert 9 == space.unwrap(
-            api.PyNumber_Power(space.wrap(3), space.wrap(2), space.w_None))
+            PyNumber_Power(space, space.wrap(3), space.wrap(2), space.w_None))
         assert 4 == space.unwrap(
-            api.PyNumber_Power(space.wrap(3), space.wrap(2), space.wrap(5)))
+            PyNumber_Power(space, space.wrap(3), space.wrap(2), space.wrap(5)))
         assert 9 == space.unwrap(
-            api.PyNumber_InPlacePower(space.wrap(3), space.wrap(2), space.w_None))
+            PyNumber_InPlacePower(space, space.wrap(3), space.wrap(2), space.w_None))
 
 
 class AppTestCNumber(AppTestCpythonExtensionBase):
diff --git a/pypy/module/cpyext/test/test_object.py b/pypy/module/cpyext/test/test_object.py
--- a/pypy/module/cpyext/test/test_object.py
+++ b/pypy/module/cpyext/test/test_object.py
@@ -1,10 +1,16 @@
-import py, pytest
+import pytest
 
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.module.cpyext.api import (
     Py_LT, Py_LE, Py_NE, Py_EQ, Py_GE, Py_GT)
+from pypy.module.cpyext.object import (
+    PyObject_IsTrue, PyObject_Not, PyObject_GetAttrString,
+    PyObject_DelAttrString, PyObject_GetAttr, PyObject_DelAttr,
+    PyObject_GetItem, PyObject_RichCompareBool,
+    PyObject_IsInstance, PyObject_IsSubclass, PyObject_AsFileDescriptor,
+    PyObject_Hash)
 
 class TestObject(BaseApiTest):
     def test_IsTrue(self, space, api):
@@ -25,9 +31,10 @@
                     raise ValueError
             return C()""")
 
-        assert api.PyObject_IsTrue(w_obj) == -1
-        assert api.PyObject_Not(w_obj) == -1
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyObject_IsTrue(space, w_obj)
+        with raises_w(space, ValueError):
+            PyObject_Not(space, w_obj)
 
     def test_HasAttr(self, space, api):
         hasattr_ = lambda w_obj, name: api.PyObject_HasAttr(w_obj,
@@ -59,22 +66,21 @@
         rffi.free_charp(buf)
         assert space.unwrap(space.getattr(w_obj, space.wrap('test'))) == 20
 
-    def test_getattr(self, space, api):
+    def test_getattr(self, space):
         charp1 = rffi.str2charp("__len__")
         charp2 = rffi.str2charp("not_real")
-        assert api.PyObject_GetAttrString(space.wrap(""), charp1)
-        assert not api.PyObject_GetAttrString(space.wrap(""), charp2)
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
-        assert api.PyObject_DelAttrString(space.wrap(""), charp1) == -1
-        assert api.PyErr_Occurred() is space.w_AttributeError
-        api.PyErr_Clear()
+        assert PyObject_GetAttrString(space, space.wrap(""), charp1)
+
+        with raises_w(space, AttributeError):
+            PyObject_GetAttrString(space, space.wrap(""), charp2)
+        with raises_w(space, AttributeError):
+            PyObject_DelAttrString(space, space.wrap(""), charp1)
         rffi.free_charp(charp1)
         rffi.free_charp(charp2)
 
-        assert api.PyObject_GetAttr(space.wrap(""), space.wrap("__len__"))
-        assert api.PyObject_DelAttr(space.wrap(""), space.wrap("__len__")) == -1
-        api.PyErr_Clear()
+        assert PyObject_GetAttr(space, space.wrap(""), space.wrap("__len__"))
+        with raises_w(space, AttributeError):
+            PyObject_DelAttr(space, space.wrap(""), space.wrap("__len__"))
 
     def test_getitem(self, space, api):
         w_t = space.wrap((1, 2, 3, 4, 5))
@@ -88,9 +94,8 @@
         assert space.getitem(w_d, space.wrap("key")) is space.w_None
 
         assert api.PyObject_DelItem(w_d, space.wrap("key")) == 0
-        assert api.PyObject_GetItem(w_d, space.wrap("key")) is None
-        assert api.PyErr_Occurred() is space.w_KeyError
-        api.PyErr_Clear()
+        with raises_w(space, KeyError):
+            PyObject_GetItem(space, w_d, space.wrap("key"))
 
     def test_size(self, space, api):
         assert api.PyObject_Size(space.newlist([space.w_None])) == 1
@@ -119,9 +124,9 @@
             w_o2 = space.wrap(o2)
 
             for opid, expected in [
-                    (Py_LT, o1 <  o2), (Py_LE, o1 <= o2),
+                    (Py_LT, o1 < o2), (Py_LE, o1 <= o2),
                     (Py_NE, o1 != o2), (Py_EQ, o1 == o2),
-                    (Py_GT, o1 >  o2), (Py_GE, o1 >= o2)]:
+                    (Py_GT, o1 > o2), (Py_GE, o1 >= o2)]:
                 assert compare(w_o1, w_o2, opid) == expected
 
         test_compare(1, 2)
@@ -129,9 +134,8 @@
         test_compare('2', '1')
 
         w_i = space.wrap(1)
-        assert api.PyObject_RichCompareBool(w_i, w_i, 123456) == -1
-        assert api.PyErr_Occurred() is space.w_SystemError
-        api.PyErr_Clear()
+        with raises_w(space, SystemError):
+            PyObject_RichCompareBool(space, w_i, w_i, 123456)
 
     def test_IsInstance(self, space, api):
         assert api.PyObject_IsInstance(space.wrap(1), space.w_int) == 1
@@ -140,8 +144,8 @@
         assert api.PyObject_IsInstance(
             space.wrap(1), space.newtuple([space.w_int, space.w_float])) == 1
         assert api.PyObject_IsInstance(space.w_type, space.w_type) == 1
-        assert api.PyObject_IsInstance(space.wrap(1), space.w_None) == -1
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_IsInstance(space, space.wrap(1), space.w_None)
 
     def test_IsSubclass(self, space, api):
         assert api.PyObject_IsSubclass(space.w_type, space.w_type) == 1
@@ -149,14 +153,13 @@
         assert api.PyObject_IsSubclass(space.w_object, space.w_type) == 0
         assert api.PyObject_IsSubclass(
             space.w_type, space.newtuple([space.w_int, space.w_type])) == 1
-        assert api.PyObject_IsSubclass(space.wrap(1), space.w_type) == -1
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_IsSubclass(space, space.wrap(1), space.w_type)
 
     def test_fileno(self, space, api):
         assert api.PyObject_AsFileDescriptor(space.wrap(1)) == 1
-        assert api.PyObject_AsFileDescriptor(space.wrap(-20)) == -1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyObject_AsFileDescriptor(space, space.wrap(-20))
 
         w_File = space.appexec([], """():
             class File:
@@ -169,9 +172,8 @@
     def test_hash(self, space, api):
         assert api.PyObject_Hash(space.wrap(72)) == 72
         assert api.PyObject_Hash(space.wrap(-1)) == -2
-        assert (api.PyObject_Hash(space.wrap([])) == -1 and
-            api.PyErr_Occurred() is space.w_TypeError)
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyObject_Hash(space, space.wrap([]))
 
     def test_hash_double(self, space, api):
         assert api._Py_HashDouble(72.0) == 72
@@ -195,7 +197,7 @@
         from pypy.interpreter import gateway
 
         AppTestCpythonExtensionBase.setup_class.im_func(cls)
-        tmpname = str(py.test.ensuretemp('out', dir=0))
+        tmpname = str(pytest.ensuretemp('out', dir=0))
         cls.w_tmpname = cls.space.wrap(tmpname)
 
         if not cls.runappdirect:
diff --git a/pypy/module/cpyext/test/test_pyerrors.py b/pypy/module/cpyext/test/test_pyerrors.py
--- a/pypy/module/cpyext/test/test_pyerrors.py
+++ b/pypy/module/cpyext/test/test_pyerrors.py
@@ -5,9 +5,7 @@
 from pypy.module.cpyext.state import State
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, ll2ctypes
-
-from pypy.interpreter.gateway import interp2app
+from rpython.rtyper.lltypesystem import rffi
 
 class TestExceptions(BaseApiTest):
     def test_GivenExceptionMatches(self, space, api):
@@ -61,13 +59,6 @@
         assert space.eq_w(state.operror.w_type, space.w_MemoryError)
         api.PyErr_Clear()
 
-    def test_BadArgument(self, space, api):
-        ret = api.PyErr_BadArgument()
-        state = space.fromcache(State)
-        assert space.eq_w(state.operror.w_type, space.w_TypeError)
-        assert ret == 0
-        api.PyErr_Clear()
-
     def test_Warning(self, space, api, capfd):
         message = rffi.str2charp("this is a warning")
         api.PyErr_WarnEx(None, message, 1)
diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py
--- a/pypy/module/cpyext/test/test_pyfile.py
+++ b/pypy/module/cpyext/test/test_pyfile.py
@@ -1,8 +1,8 @@
+import pytest
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.object import Py_PRINT_RAW
 from rpython.rtyper.lltypesystem import rffi
 from rpython.tool.udir import udir
-import pytest
 
 class TestFile(BaseApiTest):
 
diff --git a/pypy/module/cpyext/test/test_pystrtod.py b/pypy/module/cpyext/test/test_pystrtod.py
--- a/pypy/module/cpyext/test/test_pystrtod.py
+++ b/pypy/module/cpyext/test/test_pystrtod.py
@@ -1,67 +1,62 @@
 import math
 
 from pypy.module.cpyext import pystrtod
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from rpython.rtyper.lltypesystem import rffi
 from rpython.rtyper.lltypesystem import lltype
+from pypy.module.cpyext.pystrtod import PyOS_string_to_double
 
 
 class TestPyOS_string_to_double(BaseApiTest):
 
-    def test_simple_float(self, api):
+    def test_simple_float(self, space):
         s = rffi.str2charp('0.4')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert r == 0.4
         rffi.free_charp(s)
 
-    def test_empty_string(self, api):
+    def test_empty_string(self, space):
         s = rffi.str2charp('')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, None)
         rffi.free_charp(s)
 
-    def test_bad_string(self, api):
+    def test_bad_string(self, space):
         s = rffi.str2charp(' 0.4')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, None)
         rffi.free_charp(s)
 
-    def test_overflow_pos(self, api):
+    def test_overflow_pos(self, space):
         s = rffi.str2charp('1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert math.isinf(r)
         assert r > 0
         rffi.free_charp(s)
 
-    def test_overflow_neg(self, api):
+    def test_overflow_neg(self, space):
         s = rffi.str2charp('-1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, None)
+        r = PyOS_string_to_double(space, s, null, None)
         assert math.isinf(r)
         assert r < 0
         rffi.free_charp(s)
 
-    def test_overflow_exc(self, space, api):
+    def test_overflow_exc(self, space):
         s = rffi.str2charp('1e500')
         null = lltype.nullptr(rffi.CCHARPP.TO)
-        r = api.PyOS_string_to_double(s, null, space.w_ValueError)
-        assert r == -1.0
-        raises(ValueError)
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, null, space.w_ValueError)
         rffi.free_charp(s)
 
-    def test_endptr_number(self, api):
+    def test_endptr_number(self, space):
         s = rffi.str2charp('0.4')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
+        r = PyOS_string_to_double(space, s, endp, None)
         assert r == 0.4
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
@@ -69,10 +64,10 @@
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
-    def test_endptr_tail(self, api):
+    def test_endptr_tail(self, space):
         s = rffi.str2charp('0.4 foo')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
+        r = PyOS_string_to_double(space, s, endp, None)
         assert r == 0.4
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
@@ -80,16 +75,14 @@
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
-    def test_endptr_no_conversion(self, api):
+    def test_endptr_no_conversion(self, space):
         s = rffi.str2charp('foo')
         endp = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
-        r = api.PyOS_string_to_double(s, endp, None)
-        assert r == -1.0
-        raises(ValueError)
+        with raises_w(space, ValueError):
+            PyOS_string_to_double(space, s, endp, None)
         endp_addr = rffi.cast(rffi.LONG, endp[0])
         s_addr = rffi.cast(rffi.LONG, s)
         assert endp_addr == s_addr
-        api.PyErr_Clear()
         rffi.free_charp(s)
         lltype.free(endp, flavor='raw')
 
@@ -164,4 +157,4 @@
         r = api.PyOS_double_to_string(3.14, 'g', 3, 0, ptype)
         assert '3.14' == rffi.charp2str(r)
         assert ptype == lltype.nullptr(rffi.INTP.TO)
-        rffi.free_charp(r)
\ No newline at end of file
+        rffi.free_charp(r)
diff --git a/pypy/module/cpyext/test/test_sequence.py b/pypy/module/cpyext/test/test_sequence.py
--- a/pypy/module/cpyext/test/test_sequence.py
+++ b/pypy/module/cpyext/test/test_sequence.py
@@ -1,9 +1,12 @@
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
 from pypy.interpreter.error import OperationError
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext import sequence
-import py.test
+from pypy.module.cpyext.sequence import (
+    PySequence_Fast, PySequence_Contains, PySequence_Index,
+    PySequence_GetItem, PySequence_SetItem, PySequence_DelItem)
+
+import pytest
 
 class TestSequence(BaseApiTest):
     def test_check(self, space, api):
@@ -56,16 +59,12 @@
         w_t2 = api.PySequence_InPlaceRepeat(w_t1, 3)
         assert space.unwrap(w_t2) == [0, 1, 0, 1, 0, 1]
 
-    def test_exception(self, space, api):
+    def test_exception(self, space):
         message = rffi.str2charp("message")
-        assert not api.PySequence_Fast(space.wrap(3), message)
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
-
-        exc = raises(OperationError, sequence.PySequence_Fast,
-                     space, space.wrap(3), message)
-        assert exc.value.match(space, space.w_TypeError)
-        assert space.str_w(exc.value.get_w_value(space)) == "message"
+        with pytest.raises(OperationError) as excinfo:
+            PySequence_Fast(space, space.wrap(3), message)
+        assert excinfo.value.match(space, space.w_TypeError)
+        assert space.str_w(excinfo.value.get_w_value(space)) == "message"
         rffi.free_charp(message)
 
     def test_get_slice(self, space, api):
@@ -80,7 +79,7 @@
 
     def test_get_slice_fast(self, space, api):
         w_t = space.wrap([1, 2, 3, 4, 5])
-        api.PySequence_Fast(w_t, "foo") # converts
+        api.PySequence_Fast(w_t, "foo")  # converts
         assert space.unwrap(api.PySequence_GetSlice(w_t, 2, 4)) == [3, 4]
         assert space.unwrap(api.PySequence_GetSlice(w_t, 1, -1)) == [2, 3, 4]
 
@@ -97,13 +96,12 @@
         exc = raises(OperationError, space.next, w_iter)
         assert exc.value.match(space, space.w_StopIteration)
 
-    def test_contains(self, space, api):
+    def test_contains(self, space):
         w_t = space.wrap((1, 'ha'))
-        assert api.PySequence_Contains(w_t, space.wrap(u'ha'))
-        assert not api.PySequence_Contains(w_t, space.wrap(2))
-        assert api.PySequence_Contains(space.w_None, space.wrap(2)) == -1
-        assert api.PyErr_Occurred()
-        api.PyErr_Clear()
+        assert PySequence_Contains(space, w_t, space.wrap(u'ha'))
+        assert not PySequence_Contains(space, w_t, space.wrap(2))
+        with raises_w(space, TypeError):
+            PySequence_Contains(space, space.w_None, space.wrap(2))
 
     def test_setitem(self, space, api):
         w_value = space.wrap(42)
@@ -112,39 +110,33 @@
         result = api.PySequence_SetItem(l, 0, w_value)
         assert result != -1
         assert space.eq_w(space.getitem(l, space.wrap(0)), w_value)
-
-        self.raises(space, api, IndexError, api.PySequence_SetItem,
-                    l, 3, w_value)
+        with raises_w(space, IndexError):
+            PySequence_SetItem(space, l, 3, w_value)
 
         t = api.PyTuple_New(1)
         api.PyTuple_SetItem(t, 0, l)
-        self.raises(space, api, TypeError, api.PySequence_SetItem,
-                    t, 0, w_value)
-
-        self.raises(space, api, TypeError, api.PySequence_SetItem,
-                    space.newdict(), 0, w_value)
+        with raises_w(space, TypeError):
+            PySequence_SetItem(space, t, 0, w_value)
+        with raises_w(space, TypeError):
+            PySequence_SetItem(space, space.newdict(), 0, w_value)
 
     def test_delitem(self, space, api):
         w_l = space.wrap([1, 2, 3, 4])
-
         result = api.PySequence_DelItem(w_l, 2)
         assert result == 0
         assert space.eq_w(w_l, space.wrap([1, 2, 4]))
-
-        self.raises(space, api, IndexError, api.PySequence_DelItem,
-                    w_l, 3)
+        with raises_w(space, IndexError):
+            PySequence_DelItem(space, w_l, 3)
 
     def test_getitem(self, space, api):
         thelist = [8, 7, 6, 5, 4, 3, 2, 1]
         w_l = space.wrap(thelist)
-
         result = api.PySequence_GetItem(w_l, 4)
         assert space.is_true(space.eq(result, space.wrap(4)))
-
         result = api.PySequence_ITEM(w_l, 4)
         assert space.is_true(space.eq(result, space.wrap(4)))
-
-        self.raises(space, api, IndexError, api.PySequence_GetItem, w_l, 9000)
+        with raises_w(space, IndexError):
+            PySequence_GetItem(space, w_l, 9000)
 
     def test_index(self, space, api):
         thelist = [9, 8, 7, 6, 5, 4, 3, 2, 1]
@@ -155,10 +147,8 @@
         assert result == thelist.index(5)
 
         w_tofind = space.wrap(9001)
-        result = api.PySequence_Index(w_l, w_tofind)
-        assert result == -1
-        assert api.PyErr_Occurred() is space.w_ValueError
-        api.PyErr_Clear()
+        with raises_w(space, ValueError):
+            PySequence_Index(space, w_l, w_tofind)
 
         w_gen = space.appexec([], """():
            return (x ** 2 for x in range(40))""")
@@ -196,7 +186,7 @@
         assert space.int_w(space.len(w_l)) == 4
         assert space.int_w(space.getitem(w_l, space.wrap(1))) == 2
         assert space.int_w(space.getitem(w_l, space.wrap(0))) == 1
-        e = py.test.raises(OperationError, space.getitem, w_l, space.wrap(15))
+        e = pytest.raises(OperationError, space.getitem, w_l, space.wrap(15))
         assert "list index out of range" in e.value.errorstr(space)
         assert space.int_w(space.getitem(w_l, space.wrap(-1))) == 4
         space.setitem(w_l, space.wrap(1), space.wrap(13))
diff --git a/pypy/module/cpyext/test/test_setobject.py b/pypy/module/cpyext/test/test_setobject.py
--- a/pypy/module/cpyext/test/test_setobject.py
+++ b/pypy/module/cpyext/test/test_setobject.py
@@ -1,31 +1,30 @@
-import py
-
-from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from pypy.module.cpyext.setobject import (
+    PySet_Check, PyFrozenSet_Check, PyFrozenSet_CheckExact,
+    PySet_Add, PySet_Size, PySet_GET_SIZE)
 
 
 class TestTupleObject(BaseApiTest):
-    def test_setobj(self, space, api):
-        assert not api.PySet_Check(space.w_None)
-        assert not api.PyFrozenSet_Check(space.w_None)
-        assert api.PySet_Add(space.w_None, space.w_None) == -1
-        api.PyErr_Clear()
+    def test_setobj(self, space):
+        assert not PySet_Check(space, space.w_None)
+        assert not PyFrozenSet_Check(space, space.w_None)
+        with raises_w(space, SystemError):
+            PySet_Add(space, space.w_None, space.w_None)
         w_set = space.call_function(space.w_set)
-        assert not api.PyFrozenSet_CheckExact(w_set)
-        space.call_method(w_set, 'update', space.wrap([1,2,3,4]))
-        assert api.PySet_Size(w_set) == 4
-        assert api.PySet_GET_SIZE(w_set) == 4
-        raises(TypeError, api.PySet_Size(space.newlist([])))
-        api.PyErr_Clear()
+        assert not PyFrozenSet_CheckExact(space, w_set)
+        space.call_method(w_set, 'update', space.wrap([1, 2, 3, 4]))
+        assert PySet_Size(space, w_set) == 4
+        assert PySet_GET_SIZE(space, w_set) == 4
+        with raises_w(space, TypeError):
+            PySet_Size(space, space.newlist([]))
 
     def test_set_add_discard(self, space, api):
         w_set = api.PySet_New(None)
         assert api.PySet_Size(w_set) == 0
-        w_set = api.PyFrozenSet_New(space.wrap([1,2,3,4]))
+        w_set = api.PyFrozenSet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Size(w_set) == 4
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Size(w_set) == 4
         api.PySet_Add(w_set, space.wrap(6))
         assert api.PySet_Size(w_set) == 5
@@ -33,14 +32,14 @@
         assert api.PySet_Size(w_set) == 4
 
     def test_set_contains(self, space, api):
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         assert api.PySet_Contains(w_set, space.wrap(1))
         assert not api.PySet_Contains(w_set, space.wrap(0))
 
     def test_set_pop_clear(self, space, api):
-        w_set = api.PySet_New(space.wrap([1,2,3,4]))
+        w_set = api.PySet_New(space.wrap([1, 2, 3, 4]))
         w_obj = api.PySet_Pop(w_set)
-        assert space.int_w(w_obj) in (1,2,3,4)
+        assert space.int_w(w_obj) in (1, 2, 3, 4)
         assert space.len_w(w_set) == 3
         api.PySet_Clear(w_set)
         assert space.len_w(w_set) == 0
@@ -72,6 +71,5 @@
              PySet_GET_SIZE(dumb_pointer);
 
              return o;
-             """
-            )
+             """)
         ])
diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py
--- a/pypy/module/cpyext/test/test_sliceobject.py
+++ b/pypy/module/cpyext/test/test_sliceobject.py
@@ -2,13 +2,14 @@
 from pypy.module.cpyext.test.test_api import BaseApiTest
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from pypy.module.cpyext.api import Py_ssize_t, Py_ssize_tP
+from pypy.module.cpyext.sliceobject import PySlice_Check
 
 class TestSliceObject(BaseApiTest):
-    def test_slice(self, space, api):
+    def test_slice(self, space):
         w_i = space.wrap(10)
         w_slice = space.newslice(w_i, w_i, w_i)
-        assert api.PySlice_Check(w_slice)
-        assert not api.PySlice_Check(w_i)
+        assert PySlice_Check(space, w_slice)
+        assert not PySlice_Check(space, w_i)
 
     def test_GetIndicesEx(self, space, api):
         w = space.wrap
diff --git a/pypy/module/cpyext/test/test_tupleobject.py b/pypy/module/cpyext/test/test_tupleobject.py
--- a/pypy/module/cpyext/test/test_tupleobject.py
+++ b/pypy/module/cpyext/test/test_tupleobject.py
@@ -1,24 +1,25 @@
 import py
 
 from pypy.module.cpyext.pyobject import PyObject, PyObjectP, make_ref, from_ref
-from pypy.module.cpyext.tupleobject import PyTupleObject
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.rlib.debug import FatalError
+from pypy.module.cpyext.tupleobject import (
+    PyTupleObject, PyTuple_Check, PyTuple_SetItem, PyTuple_Size)
 
 
 class TestTupleObject(BaseApiTest):
 
-    def test_tupleobject(self, space, api):
-        assert not api.PyTuple_Check(space.w_None)
-        assert api.PyTuple_SetItem(space.w_None, 0, space.w_None) == -1
+    def test_tupleobject(self, space):
+        assert not PyTuple_Check(space, space.w_None)
+        with raises_w(space, SystemError):
+            PyTuple_SetItem(space, space.w_None, 0, space.w_None)
         atuple = space.newtuple([space.wrap(0), space.wrap(1),
                                  space.wrap('yay')])
-        assert api.PyTuple_Size(atuple) == 3
-        #assert api.PyTuple_GET_SIZE(atuple) == 3  --- now a C macro
-        raises(TypeError, api.PyTuple_Size(space.newlist([])))
-        api.PyErr_Clear()
+        assert PyTuple_Size(space, atuple) == 3
+        with raises_w(space, SystemError):
+            PyTuple_Size(space, space.newlist([]))
 
     def test_tuple_realize_refuses_nulls(self, space, api):
         py_tuple = api.PyTuple_New(1)
diff --git a/pypy/module/cpyext/test/test_weakref.py b/pypy/module/cpyext/test/test_weakref.py
--- a/pypy/module/cpyext/test/test_weakref.py
+++ b/pypy/module/cpyext/test/test_weakref.py
@@ -1,5 +1,6 @@
 from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from pypy.module.cpyext.test.test_api import BaseApiTest
+from pypy.module.cpyext.test.test_api import BaseApiTest, raises_w
+from pypy.module.cpyext.weakrefobject import PyWeakref_NewRef
 
 class TestWeakReference(BaseApiTest):
     def test_weakref(self, space, api):
@@ -10,12 +11,11 @@
         assert space.is_w(api.PyWeakref_LockObject(w_ref), w_obj)
 
         w_obj = space.newtuple([])
-        assert api.PyWeakref_NewRef(w_obj, space.w_None) is None
-        assert api.PyErr_Occurred() is space.w_TypeError
-        api.PyErr_Clear()
+        with raises_w(space, TypeError):
+            PyWeakref_NewRef(space, w_obj, space.w_None)
 
     def test_proxy(self, space, api):
-        w_obj = space.w_Warning # some weakrefable object
+        w_obj = space.w_Warning  # some weakrefable object
         w_proxy = api.PyWeakref_NewProxy(w_obj, None)
         assert space.unwrap(space.str(w_proxy)) == "<class 'Warning'>"
         assert space.unwrap(space.repr(w_proxy)).startswith('<weak')
diff --git a/pypy/module/test_lib_pypy/test_stackless.py b/pypy/module/test_lib_pypy/test_stackless.py
--- a/pypy/module/test_lib_pypy/test_stackless.py
+++ b/pypy/module/test_lib_pypy/test_stackless.py
@@ -599,4 +599,23 @@
 
         stackless.run()
 
-
+    def test_kill_tasklet_waiting_for_channel(self):
+        # issue #2595
+        c = stackless.channel()
+        def sender():
+            c.send(1)
+        def receiver():
+            v = c.receive()
+        def killer(tl):
+            tl.kill()
+        def main():
+            trk = stackless.tasklet(receiver)()
+            stackless.schedule()
+            killer(trk)
+            stackless.schedule()
+            stackless.tasklet(sender)()
+            stackless.schedule()
+            stackless.tasklet(receiver)()
+            stackless.schedule()
+        stackless.tasklet(main)()
+        stackless.run()
diff --git a/rpython/rlib/test/test_rposix.py b/rpython/rlib/test/test_rposix.py
--- a/rpython/rlib/test/test_rposix.py
+++ b/rpython/rlib/test/test_rposix.py
@@ -753,6 +753,9 @@
 
 @rposix_requires('posix_fadvise')
 def test_posix_fadvise():
+    if sys.maxint <= 2**32:
+        py.test.skip("ll2ctypes run of posix_fadvise() on 32-bit "
+                     "gets confused by the size of OFF_T")
     fname = str(udir.join('test_os_posix_fadvise'))
     fd = os.open(fname, os.O_CREAT | os.O_RDWR)
     try:
@@ -769,6 +772,9 @@
 
 @rposix_requires('posix_fallocate')
 def test_posix_fallocate():
+    if sys.maxint <= 2**32:
+        py.test.skip("ll2ctypes run of posix_fallocate() on 32-bit "
+                     "gets confused by the size of OFF_T")
     fname = str(udir.join('os_test.txt'))